diff --git a/.env.example b/.env.example index a5cc28273..86a2cd645 100644 --- a/.env.example +++ b/.env.example @@ -35,8 +35,10 @@ QBIT_DATA_SOURCE=qbit-data BITCOIN_DATA_SOURCE=bitcoin-data PRISM_POSTGRES_DATA_SOURCE=prism-postgres-data PRISM_POSTGRES_WAL_SOURCE=prism-postgres-wal -PRISM_POSTGRES_REPLICA_DATA_SOURCE=prism-postgres-replica-data PRISM_AUDIT_DATA_SOURCE=prism-audit-data +# Production only: host directory holding the PRISM signing seed files. It is +# mounted read-only into each mining frontend; see docs/mainnet-deployment.md. +PRISM_SECRETS_SOURCE= QBIT_CHAIN=regtest # Compose defaults this argument to regtest; mainnet deployments must set @@ -47,6 +49,9 @@ QBIT_LISTEN=0 QBIT_RPC_USER=qbitrpc QBIT_RPC_PASSWORD=change-this QBIT_RPC_PORT=18452 +# Optional complete RPC endpoint for both native Prism services, including +# HTTPS or a proxy path. Empty uses QBIT_RPC_HOST/QBIT_RPC_PORT. +QBIT_RPC_URL= QBIT_P2P_PORT=18460 QBIT_RPC_PORT_HOST=127.0.0.1:18452 QBIT_P2P_PORT_HOST=127.0.0.1:18460 @@ -164,107 +169,98 @@ AUXPOW_MINER_PASSWORD=x AUXPOW_REAL_MINER_THREADS=1 AUXPOW_REAL_MINER_TIMEOUT_SECONDS=120 -# Direct PRISM Stratum. Production should use Postgres, real signing seeds, and -# a trusted ledger public key distributed out of band from audit bundles. +# Native Rust PRISM. Every instance connects to the same PostgreSQL writer endpoint. +# Stop Python coordinators before enabling the migration; existing ledger rows remain valid. +# For an external HA database use compose.prism-external-db.yaml and set PRISM_DATABASE_URL. +# Each process receives a UUID unless an explicit, unique PRISM_INSTANCE_ID is configured. +PRISM_INSTANCE_ID= +# Empty chooses available CPU parallelism; builder threads perform CPU work concurrently. +PRISM_RUNTIME_WORKERS= +PRISM_DATABASE_MAX_CONNECTIONS=16 +PRISM_DATABASE_STATEMENT_TIMEOUT_MS=15000 +PRISM_DATABASE_LOCK_TIMEOUT_MS=5000 +PRISM_DATABASE_ACQUIRE_PROBE_IDLE_MS=100 +PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15 +PRISM_STRATUM_MAX_MESSAGE_BYTES=16384 +# CTV workers coordinate claims in PostgreSQL across all instances. +PRISM_CTV_BROADCASTER_POLL_SECONDS=30 +# Difficulty values below are for regtest. Production requires explicit non-lab values. +# Legacy *_SATS payout options remain aliases for *_BITS options. +# Python worker leases, psql subprocess options, share commit queues, and scheduler tuning +# are obsolete. Accounts/audit snapshots now live in PostgreSQL; the audit mount reads legacy bodies. PRISM_STRATUM_BIND=0.0.0.0 PRISM_STRATUM_PORT=3340 PRISM_STRATUM_PORT_HOST=3340 PRISM_AUDIT_BIND=127.0.0.1 PRISM_AUDIT_PORT=3341 PRISM_AUDIT_DIR=/var/lib/qbit-prism/audit -PRISM_AUDIT_SHARE_SEGMENT_SIZE=10000 -PRISM_AUDIT_LIVE_BUNDLE_RETENTION=5 -PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS=86400 -PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT=20 -PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS=300 -PRISM_EVIDENCE_PATH=/var/lib/qbit-prism/audit/prism-live-evidence.json -# Allocator telemetry on /metrics (issue #226). 1 reads glibc mallinfo2 once -# per scrape; 0 renders the allocator gauges unavailable and skips the call. -# The interpreter and component gauges are unconditional either way. -PRISM_MALLOC_TELEMETRY=1 -# glibc malloc arena cap for the coordinator process (issue #226). The image -# defaults to 2 and compose passes this value through to override it; read -# docs/mainnet-deployment.md "PRISM Allocator Settings" before changing it, -# and never leave it empty (glibc reads "" as 0). -MALLOC_ARENA_MAX=2 -# Operator-triggered heap census (SIGUSR1) and malloc_trim (SIGRTMIN+1), -# issue #226 part 2. All off by default; read docs/prism-capacity-readiness.md -# "Heap Census, Allocator Control, and the Resident-Set Bound" before enabling -# any of them. Reports land under PRISM_AUDIT_DIR/heap-census unless -# PRISM_HEAP_CENSUS_DIR is set. -PRISM_HEAP_CENSUS=0 -PRISM_HEAP_CENSUS_TOP_N=40 -PRISM_HEAP_CENSUS_MAX_BYTES=262144 -PRISM_HEAP_CENSUS_MAX_SECONDS=10 -PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS=60 -PRISM_HEAP_CENSUS_TRACEMALLOC=0 -PRISM_MALLOC_TRIM_SIGNAL=0 -PRISM_MALLOC_TRIM_INTERVAL_SECONDS=0 -PRISM_STOP_AFTER_BLOCK=0 -PRISM_MAX_BLOCKS=2147483647 PRISM_BLOCKPOLL_SECONDS=2 +PRISM_HASHRATE_ROLLUP_ENABLED=1 +PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS=15 +PRISM_HASHRATE_ROLLUP_BATCH_SHARES=50000 +# Empty share ledger partitions kept attached ahead of the sequence are +# maintained on this cadence by every frontend; whole seconds, 1 to 86400. +PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS=60 +PRISM_MIN_PEERS=1 +PRISM_TEMPLATE_MAX_AGE_SECONDS=120 +# Published submit authority; zero restores a live tip RPC for each share. +PRISM_SUBMIT_TIP_MAX_AGE_SECONDS=10 +# Maximum extension while a detected replacement is being prepared. +PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS=120 PRISM_BLOCKWAIT_ENABLED=1 -PRISM_BLOCKWAIT_TIMEOUT_SECONDS=5 -# Found-block fast lane: submitblock runs before writer admission/accounting. +# General node reads and wallet RPCs use their own deadline. The short submit +# deadline applies only to submitblock; ambiguous results recover from the outbox. +PRISM_RPC_TIMEOUT_SECONDS=15 PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS=1 -# Fresh deadline for each submitter Postgres statement and local DB gate -# (cheap outbox polls and fast-lane-adjacent calls only). -PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS=1 -# Startup and ancestor re-drive payloads per cursor page (1..1024). -# Set to 1 as a temporary mitigation for large candidate JSON; see #255. -# Smaller pages add DB/RPC round trips and do not bound one candidate's bytes. -PRISM_BLOCK_REPLAY_PAGE_SIZE=1024 -# Landing-class deadline for the post-acceptance accounting tail (persist, -# prior balances, confirm, prepared-state rejection). Must stay well above -# the poll budget; see issue #188. -PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS=30 -# Cap for the escalated landing deadline on timed-out landing retries. -PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS=120 -# Fail-closed bound on landed-but-unresolved accepted-parent transitions. -PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX=8 -# Bound candidate cleanup retry records and their retained terminal fences. -# Must be positive and no greater than 8192; lower to constrain overload memory. -PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX=4096 -# Emit the contended lock and current submitter phase at this cadence. -PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS=5 -# Exit nonzero when a timeout-ignoring RPC/DB worker pool stays exhausted this long. -PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS=30 -PRISM_REORG_RECONCILER_ENABLED=1 +# Rehearsal kill switch (#291): 0 keeps every found block pending and never +# calls submitblock, and keeps the CTV broadcaster from starting. Leave it at +# 1 in production. +PRISM_BLOCK_SUBMIT_ENABLED=1 +# Confirmations on a competing block before an offered candidate is settled orphaned (1..1000). +PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS=6 +PRISM_CAPTURE_OVERPAY_CEILING_BPS=100 +# #529, HA only: the dedicated failover standby's application_name. A found +# block's offer then waits up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (0 = off, +# at most 10000) for that standby to flush the block's rows; it is offered +# anyway at the bound. The writer role needs pg_monitor. Empty = no wait. +PRISM_OFFER_STANDBY_APPLICATION_NAME= +PRISM_OFFER_STANDBY_FLUSH_WAIT_MS=250 PRISM_VERSION_ROLLING_MASK=1fffe000 PRISM_COINBASE_TAG=/PRISM/ -# The /public/v1 read surface runs in its own process (the prism-public-api -# compose service, `python3 -m lab.prism.public_read_service`). Every -# PRISM_PUBLIC_* knob below is read by that service, not by prism-coordinator. -# PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS (further down) is the one exception: -# it is read by the ledger, and both processes construct one. +# The public read tier runs as `qbit-prism-server public-api` with its own +# PostgreSQL pool. Shared PRISM_PUBLIC_* metadata/cache settings also apply to +# the combined coordinator listener. PRISM_PUBLIC_API_BIND=0.0.0.0 PRISM_PUBLIC_API_PORT=3342 +PRISM_PUBLIC_RUNTIME_WORKERS=2 +PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=20 +PRISM_POSTGRES_READ_CONCURRENCY=4 +# Content-addressed audit artifacts (/public/v1/artifacts/ of a block's +# audit) are rebuilt from the share window or decoded from sealed bytes, each +# holding the whole window in memory. At most this many run at once, whatever +# PRISM_POSTGRES_READ_CONCURRENCY is (1..64). +PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY=1 +# Audit artifact requests running or waiting for a rebuild; the next distinct +# one gets 503 audit_artifact_busy with Retry-After at once (1..4096). +PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT=32 PRISM_PUBLIC_API_PORT_HOST=3342 -# Bound accepted public HTTP connections; excess connections are closed. -PRISM_PUBLIC_HTTP_MAX_CONNECTIONS=64 -# Total request-header deadline and socket I/O idle timeout. One response per connection. -PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS=10 # How often the public read service re-checks that it can reach Postgres for # its own /healthz. Its answer goes stale after max(3 * this, 15) seconds. # The same probe carries the replication check below, so it is also how often # the replica contract is re-evaluated. PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS=5 -# Per-request database deadline across all origin reads. Set 0 to disable, -# or choose a shorter deadline to finish before the fronting proxy times out. -PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=20 -# Read-only DSN for the standby the public tier serves from, using the -# PRISM_POSTGRES_* credentials the base backup cloned from the primary. This is -# deliberately not PRISM_DATABASE_URL: that one names the primary, and the -# coordinator needs it to. -# Leave empty for Compose to derive the replica URL from PRISM_POSTGRES_USER, -# PRISM_POSTGRES_PASSWORD, and PRISM_POSTGRES_DB. Set an explicit URL for a -# separately provisioned standby or different read credentials. -PRISM_PUBLIC_DATABASE_URL= -# Optional public-read psql command. Empty derives it from the public DSN -# above. This is separate from PRISM_POSTGRES_PSQL_COMMAND for the primary; -# use a replica command here when PRISM_POSTGRES_NATIVE_CLIENT=psql needs -# a custom wrapper or connection arguments. -PRISM_PUBLIC_PSQL_COMMAND= +# DSN for the standby the public tier serves from. This example uses the lab +# bootstrap role; production should use a dedicated reader with its own password. +# This is deliberately not PRISM_DATABASE_URL, which names the writer endpoint. +# See docs/prism-postgres-replica.md#public-reader-credentials for grants and +# authentication. URL-encode special characters in a password inside the DSN. +PRISM_PUBLIC_DATABASE_URL=postgresql://qbit:change-this@prism-postgres-replica:5432/qbit +# Optional password for a public-reader DSN that omits its password. Compose +# passes only this value as the public container's PGPASSWORD; it never falls +# back to PRISM_POSTGRES_PASSWORD. A password in the DSN takes precedence. +# Set the passwordless reader DSN too: this carrier alone does not select a role. +# Single-quote raw .env passwords to keep $ and # literal (do not URL-encode). +PRISM_PUBLIC_POSTGRES_PASSWORD='' # require = refuse replica-backed routes unless the backing server is in # recovery and its replication stream is live within the bound below; off = # serve whatever the DSN points at, which is the pre-replica behaviour. Leave @@ -277,31 +273,22 @@ PRISM_PUBLIC_REPLICA_MODE=require PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS=60 PRISM_PUBLIC_POOL_NAME=PRISM PRISM_PUBLIC_STRATUM_HOST= -# REQUIRED for make up-prism-pool and prism-public-api. The API runs no -# Stratum listener, so it cannot infer the pool's endpoint. Left empty it -# refuses to start rather than +# REQUIRED for prism-public-api: that process runs no Stratum listener, so it +# cannot infer the pool's endpoint. Left empty it refuses to start rather than # advertise 127.0.0.1 to miners as the pool's Stratum URL. PRISM_PUBLIC_STRATUM_URL= -# Optional external URL for the high-diff endpoint in /public/v1/mining-configuration. -# Used only when PRISM_STRATUM_HIGHDIFF_PORT is set; empty derives from -# PRISM_PUBLIC_STRATUM_URL first, then PRISM_PUBLIC_STRATUM_HOST, plus -# PRISM_STRATUM_HIGHDIFF_PORT. PRISM_PUBLIC_STRATUM_HIGHDIFF_URL= PRISM_PUBLIC_POOL_FEE_BPS=0 PRISM_PUBLIC_CONFIGURATION_LABEL="PRISM default" PRISM_PUBLIC_CONFIGURATION_DESCRIPTION= PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY="pool-selected qbit block template with PRISM payout settlement" PRISM_PUBLIC_MINIMUM_PAYOUT_BITS= -# Trailing window (seconds) used to steady /public/v1/hashrate-series points on -# sub-window buckets (5m); 0 serves raw per-bucket rates. Clamped to [0, 86400]. PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS=1800 PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX= PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX= PRISM_PUBLIC_CACHE_ENABLED=1 PRISM_PUBLIC_CACHE_TTL_SECONDS=5 PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS=30 -# Pool-wide aggregate endpoints (pool-summary, hashrate-series, miner workers) -# are expensive to recompute and tolerate more staleness than row reads. PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS=30 PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS=30 PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS=300 @@ -313,17 +300,7 @@ PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES=1048576 PRISM_PUBLIC_CACHE_DEBUG_HEADERS=0 PRISM_STRATUM_EXTRANONCE2_SIZE=8 PRISM_STRATUM_SHARE_DIFF=0.000000001 -PRISM_STRATUM_SHARE_WEIGHT=1 -PRISM_STRATUM_SHARE_WEIGHTS_JSON= PRISM_STRATUM_VARDIFF=1 -# The 1e-9 values below are for local labs. Production requires reviewed, -# explicit non-lab difficulty values. -# Release provenance (digest-qualified *_IMAGE references and absolute -# *_DATA_SOURCE host paths) is always enforced when QBIT_CHAIN=mainnet; no -# variable can disable it there. Set this to 1 to hold another chain to the same -# contract. Leave it 0 for production pools that build images from the pinned -# source (for example a public testnet4 deployment), which keeps every -# behavioral production check active. QBIT_REQUIRE_RELEASE_PROVENANCE=0 PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15 PRISM_STRATUM_VARDIFF_MIN_DIFF=0.000000001 @@ -333,204 +310,73 @@ PRISM_STRATUM_VARDIFF_MAX_STEP_UP=4 PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN=4 PRISM_STRATUM_VARDIFF_START_DIFF=0.000000001 PRISM_STRATUM_VARDIFF_EWMA_ALPHA=0.4 -PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE=0.25 -# Fast-arrival initial convergence. Credit is target-based: a session is paid -# for the difficulty its job was stamped at, so every retarget interval spent -# climbing toward the difficulty it is really working at is credit the miner -# never receives. A rental router that filters submissions at its own, much -# higher difficulty makes this acute -- the pool sees only the survivors, so -# the ordinary MAX_STEP_UP ladder needs several full RETARGET_SECONDS windows -# to catch up. These let ONE early move per connection, before the ordinary -# interval and with a wider step bound, close that gap. Set to 0 to restore -# the previous behavior exactly. +# Persist accepted-work evidence across reconnects, listeners, and frontends. +PRISM_STRATUM_VARDIFF_RESUME=1 +PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS=900 +PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES=8192 +PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR=1024 PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE=1 -# Step bound for that one initial move; must be >= PRISM_STRATUM_VARDIFF_MAX_STEP_UP, -# which every later retarget goes back to. 64 is what lets a 16384 start cross -# the 500000 a NiceHash-style router filters at in a single move. PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP=64 -# Accepted shares of evidence required before the early move is allowed. PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES=8 -# Minimum observed/current difficulty gap required. This is the noise guard: -# at 8 shares an exactly-on-target miner still has a ~35% spread on its -# observed rate, so demanding a 4x gap keeps an ordinary miner from being -# pushed up merely because this path exists, and makes the early move -# upward-only. Lower it only with evidence. PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP=4 -# Minimum window length before an early observation counts, so a burst -# delivered in a near-zero interval cannot divide real work by an almost-zero -# elapsed time and explode the observed rate. PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS=1 -# Observation only: the difficulty at which qbit_prism_vardiff_high_diff_arrival_* -# records how long, and how many accepted shares, a session needed to arrive. -# Never gates or clamps a retarget. Defaults to the high-diff listener floor. -# Set it inside the listener's MAX_DIFF or the arrival histograms stay empty. -PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF=500000 -# Set to 0 to disable idle zero-share vardiff step-down sweeps. -PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=15 -# Reconnect difficulty resume: a reconnecting worker (same listener + exact -# username) starts at its last converged difficulty instead of the lane -# start, so a mass reconnect does not re-flood the share path while every -# session re-climbs. 0 restores lane-start behavior exactly. -# LIMITATION: retention is keyed by (listener, exact Stratum username) and is -# last-writer-wins, so heterogeneous rigs that deliberately share one username -# share one retained value: a small rig reconnecting after a big one can adopt -# the big rig's converged difficulty (and vice versa) until its own shares -# retarget it. The adopted value is still bounded by the lane clamp below. -# Give each rig its own worker name (address.rig) to keep the values separate; -# PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME bounds how many connections can -# share one identity in the first place. -# With the PostgreSQL ledger, accepted-share evidence is also retained in -# qbit_worker_difficulty and preloaded before listeners accept, so recent -# converged values survive coordinator restarts. Non-PostgreSQL backends keep -# only the process-local cache and therefore start empty after a restart. -# The TTL is measured from accepted-share evidence recorded at a committed -# retarget or orderly disconnect. A long-running session that no longer -# retargets can therefore age out before an abrupt process exit even while it -# remains connected; this is bounded reconnect assistance, not an active- -# session lease or a guarantee that every restart resumes every worker. -PRISM_STRATUM_VARDIFF_RESUME=1 -# How long a retained difficulty stays adoptable. WARNING: when -# PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed) there is no idle -# step-down sweep, so a session resumed above its true hashrate can only be -# corrected by an accepted share; keep this TTL short enough that retained -# values still reflect recent hashrate. The TTL always measures the age of the -# converged value itself: reconnecting and disconnecting without submitting a -# share does not extend it. -PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS=900 -# Bound on retained per-worker entries (2x the deployed 4096 connection cap). -PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES=8192 -# Plausibility ceiling for a resumed value, as a multiple of the lane's own -# start difficulty (1024 = five 4x retarget steps). A retained value above -# min(lane max difficulty, start * factor) is clamped before adoption. -# WARNING: with PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed), a -# rig that converged high and reconnects within the TTL on weaker hardware -# can resume too high, emit no shares, and give the retargeter no signal to -# come down — stuck for that connection's life (bounded by the TTL above; -# only that miner is harmed). On hardware where converged-high rigs stall -# after resume, lower this factor. -PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR=1024 -# Seconds a share computed against the just-replaced tip is still credited -# (never block-submitted). Keep a small window that covers miner job-switch -# latency: at 0 every in-flight prior-tip share is rejected at each block, -# which miners read as pool failure. +PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE=0.25 PRISM_STRATUM_STALE_GRACE_SECONDS=3 -# Clean same-tip jobs remain valid against their original context for this -# bounded window. Set the TTL to 0 to disable; positive TTLs require positive -# per-connection caps. Production additionally requires a positive pool-wide -# connection limit, bounding retained contexts by the product of both limits. PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS=30 PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION=64 -# Staged latest-wins tip-refresh epochs. Leave disabled until rollout. -PRISM_TIP_REFRESH_EPOCH_FANOUT=0 -# Same-tip contexts retained past their connection's disconnect so a -# same-username reconnect (proxy flap) can submit in-flight work instead of -# eating unknown-job rejects. Count cap across all dead connections; entries -# keep the same-tip retention TTL. 0 restores purge-on-disconnect. -PRISM_STRATUM_DISCONNECTED_JOB_RETENTION=16384 -# Persistent ready-pool fanout workers. Values above 16 are rejected. -PRISM_TIP_REFRESH_MAX_WORKERS=16 -PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS=60 -# Reuse one exact payout window across routine generations. Normal delta -# refreshes are limited to the re-anchor cadence; found-block settlement -# bypasses the interval, and the full oracle periodically checks parity. -PRISM_PAYOUT_ARTIFACT_REUSE=1 -PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS=5 PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS=60 -PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS=60 -PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS=3600 -PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS=300 -# Route the payout-window fold, canonical digest, and incremental advance -# through the persistent --serve audit builder instead of in-process Python. -# Strictly boolean (anything else refuses startup); inert while the daemon -# transport is disabled, and any daemon anomaly still degrades to the -# in-process pipeline for that materialization without flipping this. The -# daemon carries integers in fixed widths (share_difficulty/network_difficulty -# u128, share_seq/template_height u64, ntime u32, millisecond stamps i64); a -# window holding a value beyond them is declined as out_of_range and folded -# in-process, with the daemon kept alive. -PRISM_WINDOW_PIPELINE_RUST=0 -PRISM_WORKER_METRICS_LIMIT=100 -# Coordinator maintenance of public hashrate rollups. Set 0 for raw share scans. -PRISM_HASHRATE_ROLLUP_ENABLED=1 -# Pause between caught-up passes; initial backfill runs batches back to back. -PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS=15 -PRISM_HASHRATE_ROLLUP_BATCH_SHARES=50000 -# Global thread-per-connection capacity across every PRISM listener. The -# production-facing default is above the normal 200-250 connection population; -# set it explicitly to 0 only for focused local/regtest unlimited operation. +# Connections the kernel queues on each Stratum listener before accept; a +# reconnect burst beyond it has its SYNs dropped. The kernel caps it at +# net.core.somaxconn. +PRISM_STRATUM_LISTEN_BACKLOG=4096 PRISM_STRATUM_MAX_CONNECTIONS=384 +# Zero disables the limit; retained work keeps its original username's slot. PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME=0 -# Bound authorized clients awaiting first current-tip work and expire them if -# delivery cannot make progress. A zero timeout disables expiry for focused -# tests; the pending-client bound remains positive. +# Concurrent connections allowed from one observed peer address, across both +# Stratum listeners. Zero disables the cap. Leave it at zero unless the +# deployment preserves miner source addresses: behind a terminating TCP proxy, +# NAT egress or a Docker userland proxy many miners share one address. +# See docs/prism-b4-stratum-admission.md before raising it. +PRISM_STRATUM_MAX_CONNECTIONS_PER_IP=0 +# Window the three per-session budgets below are measured over, in seconds. +PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS=60 +# Malformed frames one session may send per interval before it is +# disconnected. Frames, not bytes; zero disables the budget. +PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL=0 +# Unknown-job submits (each one ledger lookup) one session may make per +# interval before it is disconnected. Honest staleness after a tip change and +# transient resume races produce bursts here, so size it well above a miner's +# in-flight submits; zero disables the budget. +PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL=0 +# mining.authorize attempts one session may make per interval, bounding the +# address validation RPCs a single connection can drive. Zero disables it. +PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL=0 +# Job builds admitted at once per frontend. A quarter of them, at least one, +# is also the rebuild lane (no setting of its own): sessions whose work is +# still on the published parent and payout revision rebuild through it and +# keep it until the job is persisted, so the rest stay free for first jobs and +# for sessions whose work a new tip or payout revision superseded (#604). PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS=128 PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS=30 -# Dedicated first-job delivery workers. Raise under reconnect churn so first -# notifies do not queue behind each other; the pending-client bound above is -# the ceiling, since extra workers past it can never receive a task. -PRISM_INITIAL_JOB_MAX_WORKERS=4 -# Shared job-build executor threads. The scheduler admits at most two -# concurrent build flights, so values above 2 are rejected; lower to 1 to -# serialize builds on GIL-constrained single-core hosts. PRISM_JOB_BUILD_EXECUTOR_WORKERS=2 -# Interpreter thread switch interval, in seconds. Unset leaves CPython's own -# default (5ms) in force, which is what every deployment has run to date; -# setting it calls sys.setswitchinterval() before any thread starts. -# This trades share-ack tail latency against interpreter overhead, and the -# balance is host-specific -- #143's measurements moved throughput 15% and -# CPU-per-share 13% across a sweep, with the 5ms default showing the worst -# ack tail of the three settings tried. Re-measure per host with -# tests/perf/stratum_session_concurrency_load.py rather than copying a value -# between machines; a host with slower cores or a different core count will -# not share an optimum. Values above 1 are rejected at startup as a misplaced -# decimal point, and values below 1e-06 because CPython stores the interval in -# whole microseconds and would truncate them to zero -- a switch check at every -# opportunity. For the same reason a tuned value may read back rounded down to -# the nearest microsecond. -# PRISM_PYTHON_SWITCH_INTERVAL_SECONDS=0.002 -# Suppress mining-delivery health failures during bounded startup preparation. -PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS=30 -# Keep listener threads alive during transient process/system descriptor -# exhaustion instead of waiting for the watchdog to restart the coordinator. -PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS=1 -# Successful payout-address validations use a bounded LRU/TTL cache. Set either -# value to 0 to disable caching without disabling concurrent request deduping. -PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES=4096 -PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS=3600 -# Raise the coordinator's soft descriptor allowance without lowering the -# existing deployment's hard ceiling. +# Refresh builder pool threads per frontend process; empty = three quarters of +# the cores (at most 4), 0 = build serially, at most 64. +PRISM_REFRESH_BUILD_THREADS= +# After each block landing, return the heap the landing freed to the kernel +# (malloc_trim, glibc builds; #600). 0 turns it off, which lets a 400k-window +# frontend's resident set climb to about 3.3 GB instead of about 0.8 GB. +PRISM_LANDING_MALLOC_TRIM_ENABLED=1 PRISM_COORDINATOR_NOFILE_SOFT=65536 PRISM_COORDINATOR_NOFILE_HARD=524288 -# Optional second Stratum listener for rental-scale miners (e.g. NiceHash -# orders, which require a >=500000 share difficulty from the first -# mining.set_difficulty they see). Disabled unless PRISM_STRATUM_HIGHDIFF_PORT -# is set. Difficulty bounds are independent of the default listener; all other -# vardiff knobs (target/retarget/steps) inherit PRISM_STRATUM_VARDIFF_*. -# Clients may raise (never lower) their difficulty within these bounds via the -# password convention d=N,md=N or mining.suggest_difficulty. PRISM_STRATUM_HIGHDIFF_PORT= -# Host publish mapping for the high-diff listener. Leave empty while the -# listener is disabled (compose then maps an ephemeral loopback port so host -# 4334 is never occupied); set to 4334 together with -# PRISM_STRATUM_HIGHDIFF_PORT when enabling it. PRISM_STRATUM_HIGHDIFF_PORT_HOST= PRISM_STRATUM_HIGHDIFF_BIND= PRISM_STRATUM_HIGHDIFF_START_DIFF=500000 -# MIN_DIFF is a wire guarantee: stamped jobs advertise at least this value -# even while qbit network difficulty is below it (young chain, test networks), -# because marketplace verification judges the first mining.set_difficulty. PRISM_STRATUM_HIGHDIFF_MIN_DIFF=500000 PRISM_STRATUM_HIGHDIFF_MAX_DIFF=4294967296 -# Fixed difficulty when vardiff is disabled; empty tracks -# PRISM_STRATUM_HIGHDIFF_START_DIFF and must stay within [MIN_DIFF, MAX_DIFF]. PRISM_STRATUM_HIGHDIFF_SHARE_DIFF= -# Below this many distinct accepted miners, jobs run in collection mode and a -# solved block settles solver-pays-all through its signed bootstrap manifest. -PRISM_MIN_READY_MINERS=3 -# Empty means testnet/signet chains use the built-in tq fallback address, while -# regtest and mainnet keep invalid usernames fail-closed unless explicitly set. +# Used for explicitly invalid or unsupported addresses. RPC failures reject authorization. PRISM_USERNAME_FALLBACK_ADDRESS= -# Fixed absolute miner payout floor. Empty uses the formula inputs below. PRISM_PAYOUT_MIN_OUTPUT_SATS= PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES=3680 PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE=1 @@ -538,98 +384,50 @@ PRISM_PAYOUT_SAFETY_MULTIPLIER=4 PRISM_CTV_SETTLEMENT_ENABLED=0 PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT= PRISM_CTV_FANOUT_FEE_PREMIUM_BPS=12000 -PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS=2 PRISM_CTV_BROADCASTER_ENABLED= +PRISM_CTV_SPEND_SCAN_BLOCKS=32 PRISM_CTV_BROADCASTER_WALLET= PRISM_CTV_BROADCASTER_FEE_BITS=0 PRISM_CTV_BROADCASTER_LIMIT=100 -PRISM_CTV_BROADCASTER_CHUNK_SIZE=5 -PRISM_CTV_BROADCASTER_INTERVAL_SECONDS=30 -PRISM_POOL_FEE_ENABLED=0 -PRISM_POOL_FEE_BPS= -# Configure either a qbit P2MR fee address or a raw 32-byte P2MR program. +# Without a pool fee, sub-floor dust cannot be settled: once any balance is +# positive but below the payout floor, every work build fails and mining stops +# (#525). Every settlement mode, direct and CTV, therefore requires +# PRISM_POOL_FEE_ENABLED=1 with a recipient (#535); PRISM_POOL_FEE_BPS=0 is +# allowed and pays nothing until dust must be swept. +# The recipient below is a synthetic lab program nobody holds a key for, so a +# fresh lab setup is valid; production refuses it. For a real pool, set your +# own recipient: either PRISM_POOL_FEE_ADDRESS= with +# PRISM_POOL_FEE_P2MR_PROGRAM_HEX and PRISM_POOL_FEE_RECIPIENT_ID left empty, +# or your own 32-byte PRISM_POOL_FEE_P2MR_PROGRAM_HEX and a +# PRISM_POOL_FEE_RECIPIENT_ID. Exactly one of the address and the program. +PRISM_POOL_FEE_ENABLED=1 +PRISM_POOL_FEE_BPS=0 PRISM_POOL_FEE_ADDRESS= -PRISM_POOL_FEE_P2MR_PROGRAM_HEX= -PRISM_POOL_FEE_RECIPIENT_ID= +PRISM_POOL_FEE_P2MR_PROGRAM_HEX=dfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfee +PRISM_POOL_FEE_RECIPIENT_ID=dev-pool-fee PRISM_POOL_FEE_ORDER_KEY= -# Coinbase output ordering: canonical (compatibility default) or pool-fee-first. -# pool-fee-first requires an enabled pool fee, keeps the fee out of CTV fanout, -# and emits a positive fee output at coinbase vout 0. PRISM_COINBASE_OUTPUT_POLICY=canonical PRISM_POSTGRES_IMAGE=postgres:16-alpine PRISM_POSTGRES_DB=qbit PRISM_POSTGRES_USER=qbit PRISM_POSTGRES_PASSWORD=change-this -# Replica bootstrap retry budget and delay, forwarded to the standby container. -PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS=60 -PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS=5 PRISM_DATABASE_URL=postgresql://qbit:change-this@prism-postgres:5432/qbit -PRISM_POSTGRES_PSQL_COMMAND= -# Initialization applies 001_share_ledger.sql and 002_candidate_bodies.sql, -# including durable vardiff state and candidate-body storage. If disabled, -# apply both files from crates/qbit-prism/sql in that order before startup. -# The coordinator and offline recovery command refuse an unmigrated database. PRISM_POSTGRES_INIT_SCHEMA=1 -PRISM_POSTGRES_READ_CONCURRENCY=4 -# Issue #255: block-candidate bodies are written as bounded 256 KiB chunks -# (storage version 2). Set to 1 only for the compatibility/rollback release -# that must keep writing the legacy whole-jsonb candidate; the reader handles -# both versions. Once any version-2 row exists, a pre-#255 coordinator must -# not be started against the database. -PRISM_CANDIDATE_STORAGE_VERSION=2 -# Local spool directory for hydrated replay bodies (empty: the system temp -# directory) and the byte reservation that bounds concurrent spool usage. -PRISM_CANDIDATE_SPOOL_DIR= -PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES=4294967296 -# How long the server lets a coordinator session sit idle inside an open -# transaction before aborting it and releasing its locks. Bounds how long a -# vanished client (partition, SIGSTOP, VM pause) can strand the writer-lease -# row lock and block a successor's startup. -PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS=15 -# Server-side TCP keepalives toward coordinator sessions, the backstop for -# backends the idle-in-transaction timer cannot cover. On a TCP connection -# whose platform implements them, the defaults bound dead-socket detection at -# idle + count x interval = 30 + 3x10 = 60s instead of the OS default of -# roughly two hours; Unix-socket DSNs ignore them. -PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS=30 -PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS=10 -PRISM_POSTGRES_TCP_KEEPALIVES_COUNT=3 -# How long the coordinator reuses one pool-wide PRISM reward-window aggregate -# across public miner-page requests before re-running the recursive window -# scan; 0 recomputes on every request. -# Read by the ledger, so prism-coordinator and prism-public-api both set it. -# It is also the one cache stacked under another on the public surface, which -# is why /public/v1/miners/{recipient_id} carries a larger staleness budget -# (105s) than the other plain row reads (15s). -PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS=30 -PRISM_SHARE_COMMIT_BATCH_SIZE=64 -PRISM_SHARE_COMMIT_LINGER_MILLISECONDS=5 -PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15 -PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS=120 -PRISM_COORDINATION_BLOCKED_EXIT_SECONDS=900 -# How long an own-hash tip observation keeps protecting a submitted block -# candidate from terminal abandonment while instantaneous chain probes -# disagree (lost submitblock acks, transient fork views). -PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS=300 -PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS=15 PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS=15 -PRISM_LEDGER_WRITER_ID=prism-coordinator -PRISM_LEDGER_WRITER_EPOCH=1 -PRISM_LEDGER_WRITER_SESSION_TOKEN= PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX= -PRISM_LEDGER_LEASE_TTL_SECONDS=60 -# Per-attempt lock/statement deadline and attempt budget for the startup -# writer-lease acquisition. Attempts x deadline (5x5s = 25s) is sized to -# outlast PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, so an orphaned -# lock is normally reaped mid-budget and startup self-heals; an acquisition -# that never completes within the budget exits the coordinator visibly -# instead of blocking for hours. -PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS=5 -PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS=5 -# Bounds durable writer flushing before shutdown leaves the lease to its TTL. -PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS=15 PRISM_MANIFEST_SIGNING_SEED_HEX= PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX= +# Mounted-file forms of the two seeds, as paths inside the container. +# compose.production.yaml points them at PRISM_SECRETS_SOURCE and clears the +# direct values above, which production rejects. A lab may use either form, +# but not both for one seed. +PRISM_MANIFEST_SIGNING_SEED_HEX_FILE= +PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE= +# Optional bearer token for the operator listener (PRISM_AUDIT_PORT). Set at +# most one form; the built-in healthcheck sends it, and the public API never +# receives it. +PRISM_OPERATOR_BEARER_TOKEN= +PRISM_OPERATOR_BEARER_TOKEN_FILE= PRISM_ALLOW_MEMORY_LEDGER=0 PRISM_ALLOW_TEST_SIGNING_SEEDS=0 PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=0 diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..e8a0cb81d --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +# actionlint configuration: the workflow runs on Blacksmith runners, whose +# labels actionlint does not know. +self-hosted-runner: + labels: + - blacksmith-2vcpu-ubuntu-2404 + # The nightly load harness (prism-load-nightly.yml) runs its presets on + # the runner each preset names. + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-16vcpu-ubuntu-2404 + # The runner probe (prism-load-runner-probe.yml, #541) also measures the + # 32 vCPU class. + - blacksmith-32vcpu-ubuntu-2404 diff --git a/.github/actions/apt-install/action.yml b/.github/actions/apt-install/action.yml new file mode 100644 index 000000000..d1da95da2 --- /dev/null +++ b/.github/actions/apt-install/action.yml @@ -0,0 +1,56 @@ +name: Install Ubuntu packages +description: >- + Install Ubuntu packages with .github/scripts/apt-install.sh, which retries, + falls back to the mirrors in .github/apt/ubuntu-mirrors.txt, and keeps the + downloaded packages in a cache (saved with actions/cache) so an install can + still succeed while the Ubuntu mirrors are unreachable. + +inputs: + packages: + description: Space-separated Ubuntu package names. + required: true + +runs: + using: composite + steps: + - name: Compute the package cache key + id: key + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + read -r -d '' -a packages <<< "${PACKAGES}" || true + digest="$(printf '%s\n' "${packages[@]}" | sort -u | sha256sum | cut -c1-16)" + # One cache per package set and month: a new month saves a fresh set, + # and restores from the previous month until then. + prefix="apt-install-v1-${RUNNER_OS}-${RUNNER_ARCH}-${digest}-" + { + printf 'prefix=%s\n' "${prefix}" + printf 'key=%s%s\n' "${prefix}" "$(date -u +%Y-%m)" + } >> "${GITHUB_OUTPUT}" + + - name: Restore the package cache + id: restore + uses: actions/cache/restore@v6 + with: + path: ~/.cache/apt-install + key: ${{ steps.key.outputs.key }} + restore-keys: ${{ steps.key.outputs.prefix }} + + - name: Install the packages + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + read -r -d '' -a packages <<< "${PACKAGES}" || true + APT_INSTALL_CACHE_DIR="${HOME}/.cache/apt-install" \ + bash "${GITHUB_ACTION_PATH}/../../scripts/apt-install.sh" "${packages[@]}" + + - name: Save the package cache + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.cache/apt-install + key: ${{ steps.key.outputs.key }} diff --git a/.github/actions/prism-load-evidence/action.yml b/.github/actions/prism-load-evidence/action.yml new file mode 100644 index 000000000..6fea4282e --- /dev/null +++ b/.github/actions/prism-load-evidence/action.yml @@ -0,0 +1,237 @@ +name: Record PRISM load evidence +description: >- + #551 (S6 of #487): read the ci-evidence trend, apply the report-only + regression rule to this run's trend rows, and, when `write` is true, + promote the selected bundles to a release, append the rows to ci-evidence + and open or update the one regression tracking issue. With `write` false + (pull requests and dispatches) it only reads and reports: nothing is + recorded, promoted or filed. Expects this repository checked out at the + workspace root (the scripts and the regression config are read from it) + and the rows as scripts/prism_load_trend.py `row` wrote them, one *.json + per preset run. A writing job needs `contents: write` (rows, releases), + `issues: write` and `actions: read` (to download the bundles it promotes); + it must be a scheduled run or a `v*` tag push, and the first step refuses + anything else, as `append` refuses any push row whose `ref` is not + refs/tags/v*. For #550's tag job: rows built with `--lane L3 --ref + "$GITHUB_REF"`, `rows-dir` holding them, + `promote: all`, `release: ` (the release must already exist), + and `regression-config: ''` unless L3 joins the rule's lanes. + +inputs: + rows-dir: + description: Directory of this run's trend rows (*.json). + required: true + write: + description: >- + `true` to promote, append and file the issue; only on a schedule or a + `v*` tag push. + required: false + default: "false" + promote: + description: >- + Which rows' bundles to promote: `policy` (L3, L5 and the weekly soak always, L2 when + flagged), `all`, `flagged` or `none`. + required: false + default: policy + release: + description: >- + `monthly` for the rolling ci-evidence/YYYY-MM pre-release (created when + missing), or an existing release's tag (the full suite's). + required: false + default: monthly + regression-config: + description: The rule's thresholds; empty skips the rule. + required: false + default: test/prism-load-regression.toml + variance: + description: >- + #542's variance document (scripts/prism_load_probe.py variance --out). + Empty uses test/prism-load-variance.json when it is checked in, and the + provisional spread otherwise. + required: false + default: "" + issue-label: + description: The label of the one regression tracking issue. + required: false + default: prism-load-regression + artifact-name: + description: Name of the artifact the verdict is uploaded as. + required: false + default: prism-load-regression-verdict + github-token: + description: The job's token. + required: true + +outputs: + regressions: + description: Numbers the rule flagged; `unknown` when it could not run, empty when skipped. + value: ${{ steps.regress.outputs.regressions }} + +runs: + using: composite + steps: + - name: Refuse a write from an event that may not write + if: ${{ inputs.write == 'true' }} + shell: bash + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + set -euo pipefail + if [[ "${EVENT}" == schedule || ( "${EVENT}" == push && "${REF}" == refs/tags/v* ) ]]; then + exit 0 + fi + echo "::error::${EVENT} on ${REF} may not write evidence; pull request and dispatch runs produce artifacts only (#551)" + exit 1 + + - name: Read the trend from ci-evidence + id: history + shell: bash + run: | + set -uo pipefail + dir="${RUNNER_TEMP}/prism-load-history" + rm -rf "${dir}" + if python3 scripts/prism_load_trend.py fetch --out "${dir}"; then + printf 'ok=true\ndir=%s\n' "${dir}" >> "${GITHUB_OUTPUT}" + else + echo "::warning::could not read ci-evidence; the regression rule has no history this run" + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Apply the regression rule (report-only) + id: regress + if: ${{ inputs.regression-config != '' }} + shell: bash + env: + ROWS: ${{ inputs.rows-dir }} + CONFIG: ${{ inputs.regression-config }} + VARIANCE: ${{ inputs.variance }} + HISTORY_OK: ${{ steps.history.outputs.ok }} + HISTORY: ${{ steps.history.outputs.dir }} + run: | + set -uo pipefail + verdict="${RUNNER_TEMP}/prism-load-verdict" + mkdir -p "${verdict}" + if [[ "${HISTORY_OK}" != true ]]; then + # Unknown, not "nothing regressed" (EP-OBSERVABILITY). + printf 'regressions=unknown\n' >> "${GITHUB_OUTPUT}" + printf '### Load regression rule (#551)\n\nNo verdict: ci-evidence could not be read.\n' >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + args=(--history "${HISTORY}" --rows "${ROWS}" --config "${CONFIG}" + --json "${verdict}/verdict.json" --markdown "${verdict}/verdict.md" + --github-output "${GITHUB_OUTPUT}" + --repository "${GITHUB_REPOSITORY}" --server-url "${GITHUB_SERVER_URL}") + if [[ -z "${VARIANCE}" && -f test/prism-load-variance.json ]]; then + VARIANCE=test/prism-load-variance.json + fi + if [[ -n "${VARIANCE}" ]]; then + args+=(--variance "${VARIANCE}") + fi + if python3 scripts/prism_load_regress.py "${args[@]}" > /dev/null; then + cat "${verdict}/verdict.md" >> "${GITHUB_STEP_SUMMARY}" + printf 'verdict=%s\n' "${verdict}/verdict.json" >> "${GITHUB_OUTPUT}" + else + printf 'regressions=unknown\n' >> "${GITHUB_OUTPUT}" + printf '### Load regression rule (#551)\n\nNo verdict: the rule could not read its inputs (see the log).\n' >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Upload the verdict + if: ${{ steps.regress.outputs.verdict != '' }} + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + overwrite: true + retention-days: 90 + path: ${{ runner.temp }}/prism-load-verdict + if-no-files-found: error + + - name: Promote the selected bundles + id: promote + if: ${{ inputs.write == 'true' && inputs.promote != 'none' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + ROWS: ${{ inputs.rows-dir }} + SELECT: ${{ inputs.promote }} + RELEASE: ${{ inputs.release }} + VERDICT: ${{ steps.regress.outputs.verdict }} + run: | + set -uo pipefail + args=(--rows "${ROWS}" --select "${SELECT}" --release "${RELEASE}" --repository "${GITHUB_REPOSITORY}") + if [[ -n "${VERDICT}" ]]; then + args+=(--verdict "${VERDICT}") + fi + # A failed promotion is written into its row as failed and the rows + # are still recorded; the last step fails the job. + if python3 scripts/prism_load_trend.py promote "${args[@]}"; then + printf 'ok=true\n' >> "${GITHUB_OUTPUT}" + else + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Append the rows to ci-evidence + id: append + if: ${{ inputs.write == 'true' }} + shell: bash + env: + ROWS: ${{ inputs.rows-dir }} + run: | + set -uo pipefail + if python3 scripts/prism_load_trend.py append --rows "${ROWS}"; then + printf 'ok=true\n' >> "${GITHUB_OUTPUT}" + else + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Open or update the regression tracking issue + if: >- + ${{ inputs.write == 'true' && steps.regress.outputs.regressions != '' + && steps.regress.outputs.regressions != '0' && steps.regress.outputs.regressions != 'unknown' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + LABEL: ${{ inputs.issue-label }} + COUNT: ${{ steps.regress.outputs.regressions }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY}" + gh label create "${LABEL}" --repo "${repo}" --color D93F0B \ + --description "The load regression rule flagged a number (#551); report-only" 2>/dev/null || true + # An issue body holds 65,536 characters. + body="$(head -c 60000 "${RUNNER_TEMP}/prism-load-verdict/verdict.md") + + Kept by the prism-load-evidence action (#551) from ${RUN_URL}. Report-only: it does not page and does not block a merge. The body always shows the latest flagged run; each flagged run also comments. Close it once the cause is fixed or the change is accepted." + issue="$(gh issue list --repo "${repo}" --label "${LABEL}" --state open --limit 1 --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue edit "${issue}" --repo "${repo}" --body "${body}" + gh issue comment "${issue}" --repo "${repo}" \ + --body "${RUN_URL} flagged ${COUNT} number(s); the body now shows that run." + else + gh issue create --repo "${repo}" --label "${LABEL}" \ + --title "PRISM load: the regression rule flagged a number (report-only)" --body "${body}" + fi + + - name: Fail if evidence was not recorded + if: ${{ always() }} + shell: bash + env: + WRITE: ${{ inputs.write }} + HISTORY: ${{ steps.history.outputs.ok }} + REGRESSIONS: ${{ steps.regress.outputs.regressions }} + PROMOTED: ${{ steps.promote.outputs.ok }} + APPENDED: ${{ steps.append.outputs.ok }} + run: | + set -euo pipefail + failed=() + [[ "${HISTORY}" == false ]] && failed+=("reading ci-evidence") + [[ "${REGRESSIONS}" == unknown ]] && failed+=("the regression rule") + [[ "${PROMOTED}" == false ]] && failed+=("a promotion (see the rows' promotion field)") + if [[ "${WRITE}" == true && "${APPENDED}" != true ]]; then + failed+=("appending the rows to ci-evidence") + fi + if (( ${#failed[@]} )); then + printf '::error::not done: %s\n' "${failed[@]}" + exit 1 + fi diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml new file mode 100644 index 000000000..e85a67584 --- /dev/null +++ b/.github/actions/rust-toolchain/action.yml @@ -0,0 +1,22 @@ +name: Set up the pinned Rust toolchain +description: >- + Install the toolchain that rust-toolchain.toml pins and assert that the + active rustc and cargo report exactly that version. The file is the one + source of the pin; scripts/check_toolchain_pin.py holds Cargo.toml's + rust-version and the lab/prism/Dockerfile builder image to it. + +runs: + using: composite + steps: + - name: Install the toolchain named in rust-toolchain.toml + shell: bash + run: | + set -euo pipefail + # rustup 1.28 and later read rust-toolchain.toml when no toolchain is + # named, installing the pinned channel with its profile and components. + rustup toolchain install --no-self-update + rustup show active-toolchain + + - name: Assert the active toolchain is the pin + shell: bash + run: python3 scripts/check_toolchain_pin.py --installed diff --git a/.github/apt/ubuntu-mirrors.txt b/.github/apt/ubuntu-mirrors.txt new file mode 100644 index 000000000..3b7f6cfcc --- /dev/null +++ b/.github/apt/ubuntu-mirrors.txt @@ -0,0 +1,15 @@ +# Fallback Ubuntu mirrors for CI package installs (.github/scripts/apt-install.sh). +# +# The runner's own Ubuntu sources are tried first. If an install fails there, +# the helper points apt at this list instead, and apt tries these mirrors in +# priority order (without a priority, apt picks mirrors at random). Metadata +# must be separated from the URI by a tab. Azure's cloud mirror, the kernel.org mirror and Princeton's mirror +# run on infrastructure separate from Canonical's archive, which comes last. +# +# Every mirror here must carry the release, -updates, -backports and +# -security suites for amd64. apt checks everything against the Ubuntu +# archive signing key, so a mirror cannot change what gets installed. +http://azure.archive.ubuntu.com/ubuntu/ priority:1 +https://mirrors.edge.kernel.org/ubuntu/ priority:2 +https://mirror.math.princeton.edu/pub/ubuntu/ priority:3 +http://archive.ubuntu.com/ubuntu/ priority:4 diff --git a/.github/scripts/apt-install.sh b/.github/scripts/apt-install.sh new file mode 100644 index 000000000..3d24f5eea --- /dev/null +++ b/.github/scripts/apt-install.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install Ubuntu packages on a CI runner and ride out Ubuntu mirror outages. +# +# - apt retries each download itself (Acquire::Retries). +# - If the whole update-and-install still fails, for example because the +# archive refused connections or a mirror was mid-sync ("Hash Sum mismatch"), +# this clears the downloaded package lists, waits, and tries again. After the +# first failure it also replaces the runner's Ubuntu sources with the pinned +# fallback mirrors in .github/apt/ubuntu-mirrors.txt, which apt tries in order. +# - Every attempt but the last treats a partially failed `apt-get update` as a +# failure, instead of silently installing from stale indexes. The last attempt +# keeps apt's default leniency, so this is never stricter than a plain +# update-and-install. +# - With APT_INSTALL_CACHE_DIR set (the apt-install composite action sets it and +# keeps it with actions/cache), the packages a successful install downloaded +# are kept there. They seed the next install, and if every online attempt +# fails they are installed directly with dpkg instead. Nothing the runner +# already has at the same or a newer version is reinstalled or downgraded. +# +# Usage: bash .github/scripts/apt-install.sh package [package...] +# +# Settings: +# APT_INSTALL_ATTEMPTS default 4 +# APT_INSTALL_PAUSE_SECONDS default 30, multiplied by the attempt number +# APT_INSTALL_DEADLINE_SECONDS default 480; after this, the next attempt is +# the last +# APT_INSTALL_FALLBACK_MIRRORS default .github/apt/ubuntu-mirrors.txt +# APT_INSTALL_CACHE_DIR unset: no package cache + +ATTEMPTS="${APT_INSTALL_ATTEMPTS:-4}" +PAUSE_SECONDS="${APT_INSTALL_PAUSE_SECONDS:-30}" +DEADLINE_SECONDS="${APT_INSTALL_DEADLINE_SECONDS:-480}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +FALLBACK_MIRRORS="${APT_INSTALL_FALLBACK_MIRRORS:-${SCRIPT_DIR}/../apt/ubuntu-mirrors.txt}" +CACHE_DIR="${APT_INSTALL_CACHE_DIR:-}" +# apt downloads into a directory of its own, so the packages outlive the +# image's post-install clean-up and can be copied into the cache. +ARCHIVES=/var/cache/apt-install/archives + +if [[ $# -eq 0 ]]; then + printf 'apt-install: no packages given\n' >&2 + exit 2 +fi + +SUDO=() +if [[ "$(id -u)" -ne 0 ]]; then + SUDO=(sudo) +fi + +APT_OPTIONS=( + -o Acquire::Retries=5 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o "Dir::Cache::archives=${ARCHIVES}/" + -o APT::Keep-Downloaded-Packages=true +) + +shopt -s nullglob + +# Print the installed version of a package, or nothing if it is not installed. +installed_version() { + local status="" version="" + # shellcheck disable=SC2016 # dpkg-query's own ${field} syntax, not a shell expansion + read -r status version < <(dpkg-query --show \ + --showformat='${db:Status-Status} ${Version}\n' "$1" 2>/dev/null) || true + if [[ "${status}" == installed ]]; then + printf '%s' "${version}" + fi +} + +fallback_active=0 + +# Replace the runner's Ubuntu sources with the pinned fallback mirror list. +use_fallback_mirrors() { + local arch codename source + if (( fallback_active == 1 )); then + return 0 + fi + fallback_active=1 + if [[ ! -r "${FALLBACK_MIRRORS}" ]]; then + printf '::warning::apt-install: no fallback mirror list at %s\n' "${FALLBACK_MIRRORS}" + return 0 + fi + arch="$(dpkg --print-architecture)" + if [[ "${arch}" != amd64 && "${arch}" != i386 ]]; then + printf '::warning::apt-install: the fallback mirrors carry amd64 and i386 only, not %s\n' "${arch}" + return 0 + fi + # shellcheck source=/dev/null + codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")" + if [[ -z "${codename}" ]]; then + printf '::warning::apt-install: cannot tell the Ubuntu release, so the fallback mirrors are not used\n' + return 0 + fi + for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [[ -f "${source}" ]] && + grep -q -E '(archive|security)\.ubuntu\.com|mirror\+file:[^[:space:]]*ubuntu' "${source}"; then + "${SUDO[@]}" mv -- "${source}" "${source}.apt-install-disabled" + fi + done + "${SUDO[@]}" install -m 0644 -- "${FALLBACK_MIRRORS}" /etc/apt/apt-install-ubuntu-mirrors.txt + printf 'Types: deb\nURIs: mirror+file:/etc/apt/apt-install-ubuntu-mirrors.txt\nSuites: %s %s-updates %s-backports %s-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n' \ + "${codename}" "${codename}" "${codename}" "${codename}" | + "${SUDO[@]}" tee /etc/apt/sources.list.d/apt-install-fallback.sources >/dev/null + # apt retries a failed file on the same mirror before moving to the next one, + # so its growing retry delay would keep an unreachable mirror blocking every + # download for minutes. Retry immediately instead. + APT_OPTIONS+=(-o Acquire::Retries::Delay=false) + printf '::warning::apt-install: switched the Ubuntu sources to the fallback mirrors in .github/apt/ubuntu-mirrors.txt\n' +} + +# Replace the cache with exactly the downloaded packages this install used. +save_cache() { + local fresh="${CACHE_DIR}.new" deb package kept=0 + rm -rf -- "${fresh}" + mkdir -p -- "${fresh}/archives" + for deb in "${ARCHIVES}"/*.deb; do + package="$(dpkg-deb --field "${deb}" Package)" + if [[ "$(installed_version "${package}")" == "$(dpkg-deb --field "${deb}" Version)" ]]; then + cp -- "${deb}" "${fresh}/archives/" + kept=$(( kept + 1 )) + fi + done + printf 'saved %s: %d packages for %s\n' "$(date -u +%Y-%m-%dT%H:%MZ)" "${kept}" "$*" \ + > "${fresh}/manifest.txt" + rm -rf -- "${CACHE_DIR}" + mv -- "${fresh}" "${CACHE_DIR}" +} + +# Install the cached packages with dpkg, skipping any the runner already has +# at the same or a newer version, then check every requested package is there. +install_from_cache() { + local debs=() deb package version current + for deb in "${CACHE_DIR}"/archives/*.deb; do + package="$(dpkg-deb --field "${deb}" Package)" + version="$(dpkg-deb --field "${deb}" Version)" + current="$(installed_version "${package}")" + if [[ -n "${current}" ]] && dpkg --compare-versions "${current}" ge "${version}"; then + continue + fi + debs+=("${deb}") + done + printf '::warning::apt-install: the Ubuntu mirrors are unreachable; installing %d cached packages (%s)\n' \ + "${#debs[@]}" "$(head -n 1 "${CACHE_DIR}/manifest.txt" 2>/dev/null || printf 'no manifest')" + if (( ${#debs[@]} > 0 )); then + "${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive dpkg --install "${debs[@]}" || return 1 + fi + for package in "$@"; do + if [[ -z "$(installed_version "${package%%=*}")" ]]; then + return 1 + fi + done +} + +"${SUDO[@]}" mkdir -p "${ARCHIVES}/partial" +if [[ -n "${CACHE_DIR}" ]]; then + cached=("${CACHE_DIR}"/archives/*.deb) + if (( ${#cached[@]} > 0 )); then + # apt reuses a cached package whose checksum still matches the index. + "${SUDO[@]}" cp -- "${cached[@]}" "${ARCHIVES}/" + fi +fi + +attempt=1 +while true; do + last=0 + if (( attempt >= ATTEMPTS || SECONDS >= DEADLINE_SECONDS )); then + last=1 + fi + update_options=() + if (( last == 0 )); then + update_options=(--error-on=any) + fi + + if "${SUDO[@]}" apt-get "${APT_OPTIONS[@]}" "${update_options[@]}" update && + "${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive \ + apt-get "${APT_OPTIONS[@]}" install -y --no-install-recommends "$@"; then + if [[ -n "${CACHE_DIR}" ]]; then + save_cache "$@" || + printf '::warning::apt-install: could not update the package cache in %s\n' "${CACHE_DIR}" + fi + exit 0 + fi + + if (( last == 1 )); then + break + fi + + pause=$(( PAUSE_SECONDS * attempt )) + printf '::warning::apt-install: attempt %d of %d failed; clearing package lists and retrying in %ds\n' \ + "${attempt}" "${ATTEMPTS}" "${pause}" + use_fallback_mirrors + "${SUDO[@]}" find /var/lib/apt/lists -mindepth 1 -maxdepth 1 ! -name lock -exec rm -rf {} + + sleep "${pause}" + attempt=$(( attempt + 1 )) +done + +if [[ -n "${CACHE_DIR}" && -d "${CACHE_DIR}/archives" ]] && install_from_cache "$@"; then + exit 0 +fi + +printf '::error::apt-install: installing %s failed after %d attempts%s\n' \ + "$*" "${attempt}" "${CACHE_DIR:+, and the package cache could not supply it}" +exit 1 diff --git a/.github/scripts/install-pinned-tool.sh b/.github/scripts/install-pinned-tool.sh new file mode 100755 index 000000000..785034e07 --- /dev/null +++ b/.github/scripts/install-pinned-tool.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Install one pinned CI tool from its GitHub release after checking its sha256. +# +# Usage: install-pinned-tool.sh +# install-pinned-tool.sh --cache-metadata +# +# The versions and digests below are the pins; bump them together. Each digest +# is the sha256 of the release asset as published (cargo-deny also publishes a +# .sha256 file next to its tarball, which matched when the pin was set). +# +# Downloads ride out GitHub release outages: curl retries transient failures +# (timeouts, 408, 429, 5xx) every 15 seconds for up to 180 seconds, and the +# whole download is killed (SIGKILL) at 210 seconds. Other failures, such as a +# 404, fail at once. +# +# With PINNED_TOOL_CACHE_DIR set, the release asset as published is kept under +# //// and installed from there on the next +# run without any download. A cached asset must be a regular file; it is copied +# to private scratch space and checked against the pinned sha256 before anything +# is extracted, installed or run. A bad entry fails without refetching, so +# remove it by hand. +# Unset means no cache. +# +# --cache-metadata writes the cache key and paths for actions/cache to +# GITHUB_OUTPUT (cache-dir, path and key), so the pins stay in this file. It +# needs PINNED_TOOL_CACHE_DIR, RUNNER_OS and RUNNER_ARCH. +set -euo pipefail + +usage='usage: install-pinned-tool.sh + install-pinned-tool.sh --cache-metadata ' + +metadata_only=0 +if [[ "${1:-}" == --cache-metadata ]]; then + metadata_only=1 + shift + if [[ $# -ne 1 ]]; then + printf '%s\n' "${usage}" >&2 + exit 2 + fi +fi + +tool="${1:?${usage}}" +if (( metadata_only == 0 )); then + destination="${2:?${usage}}" +fi + +case "${tool}" in + cargo-deny) + version=0.20.2 + asset="cargo-deny-${version}-x86_64-unknown-linux-musl" + url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${version}/${asset}.tar.gz" + sha256=9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f + ;; + hadolint) + version=2.15.1 + url="https://github.com/hadolint/hadolint/releases/download/v${version}/hadolint-linux-x86_64" + sha256=c7187db94eeeeca956519a6af171adc31453941a1e777961f6e680f697c8c507 + ;; + *) + echo "install-pinned-tool: unknown tool ${tool}; expected cargo-deny or hadolint" >&2 + exit 2 + ;; +esac + +if [[ -n "${PINNED_TOOL_CACHE_DIR+set}" && -z "${PINNED_TOOL_CACHE_DIR}" ]]; then + echo "install-pinned-tool: PINNED_TOOL_CACHE_DIR is set but empty; unset it to disable the cache" >&2 + exit 2 +fi +cache_dir="${PINNED_TOOL_CACHE_DIR:-}" +cache_entry="" +cached_asset="" +if [[ -n "${cache_dir}" ]]; then + cache_entry="${cache_dir}/${tool}/${version}/${sha256}" + cached_asset="${cache_entry}/${url##*/}" +fi + +if (( metadata_only == 1 )); then + for name in PINNED_TOOL_CACHE_DIR RUNNER_OS RUNNER_ARCH GITHUB_OUTPUT; do + if [[ -z "${!name:-}" ]]; then + echo "install-pinned-tool: --cache-metadata needs ${name}" >&2 + exit 2 + fi + done + { + printf 'cache-dir=%s\n' "${cache_dir}" + printf 'path=%s\n' "${cache_entry}" + printf 'key=pinned-tool-v1-%s-%s-%s-%s-%s\n' \ + "${RUNNER_OS}" "${RUNNER_ARCH}" "${tool}" "${version}" "${sha256}" + } >> "${GITHUB_OUTPUT}" + exit 0 +fi + +scratch="$(mktemp -d)" +publishing="" +download_pid="" +cleanup() { + rm -rf "${scratch}" + if [[ -n "${publishing}" ]]; then + rm -f "${publishing}" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +cancel_download() { + # timeout forwards TERM to its curl process and reaps it before exiting. + # Ignore further cancellation while waiting for that cleanup to finish. + trap '' INT TERM + kill -TERM "${download_pid}" 2>/dev/null || true + wait "${download_pid}" 2>/dev/null || true + exit "$1" +} + +verify() { + printf '%s %s\n' "${sha256}" "$1" | sha256sum --check --quiet +} + +if [[ -n "${cached_asset}" && ( -e "${cached_asset}" || -L "${cached_asset}" ) ]]; then + # Only a plain file is read: a link to a FIFO or device could block or fill + # the disk before the check. + if [[ -L "${cached_asset}" || ! -f "${cached_asset}" ]]; then + echo "install-pinned-tool: cached ${cached_asset} is not a regular file; not using or refetching it, remove it and retry" >&2 + exit 1 + fi + # Check the private copy, not the shared entry, so nothing can change what + # was checked before it is used. + cp -- "${cached_asset}" "${scratch}/download" + if ! verify "${scratch}/download"; then + echo "install-pinned-tool: cached ${cached_asset} does not match the pinned sha256; not using or refetching it, remove it and retry" >&2 + exit 1 + fi + printf 'install-pinned-tool: using cached %s\n' "${cached_asset}" +else + status=0 + # Record signals during launch so cancellation cannot orphan the child + # between starting it and recording its PID. + cancelled=0 + trap 'cancelled=130' INT + trap 'cancelled=143' TERM + timeout --signal=KILL 210 \ + curl --fail --location --silent --show-error \ + --retry 12 --retry-delay 15 --retry-max-time 180 \ + --connect-timeout 10 --max-time 30 \ + --output "${scratch}/download" "${url}" & + download_pid=$! + trap 'cancel_download 130' INT + trap 'cancel_download 143' TERM + if (( cancelled != 0 )); then + cancel_download "${cancelled}" + fi + # Unlike a foreground external command, Bash's wait services traps promptly. + wait "${download_pid}" || status=$? + trap 'exit 130' INT + trap 'exit 143' TERM + download_pid="" + if (( status == 137 )); then + echo "install-pinned-tool: download of ${url} did not finish within 210 seconds" >&2 + exit 1 + elif (( status != 0 )); then + echo "install-pinned-tool: download of ${url} failed (curl exit ${status}); transient errors were retried for up to 180 seconds" >&2 + exit 1 + fi + if ! verify "${scratch}/download"; then + echo "install-pinned-tool: ${url} does not match the pinned sha256" >&2 + exit 1 + fi + if [[ -n "${cache_entry}" ]]; then + # Publish under a temporary name in the same directory, check that copy, + # then rename it into place, so readers and concurrent writers only ever + # see a whole, checked asset. + mkdir -p "${cache_entry}" + publishing="$(mktemp "${cache_entry}/.publish.XXXXXX")" + cp -- "${scratch}/download" "${publishing}" + if ! verify "${publishing}"; then + echo "install-pinned-tool: copy of ${url} into ${cache_entry} does not match the pinned sha256" >&2 + exit 1 + fi + chmod 0644 "${publishing}" + mv -fT -- "${publishing}" "${cached_asset}" + publishing="" + printf 'install-pinned-tool: cached %s\n' "${cached_asset}" + fi +fi + +mkdir -p "${destination}" +case "${tool}" in + cargo-deny) + tar -xzf "${scratch}/download" -C "${scratch}" + install -m 0755 "${scratch}/${asset}/cargo-deny" "${destination}/cargo-deny" + ;; + hadolint) + install -m 0755 "${scratch}/download" "${destination}/hadolint" + ;; +esac +printf 'install-pinned-tool: %s %s -> %s\n' "${tool}" "${version}" "${destination}/${tool}" +"${destination}/${tool}" --version diff --git a/.github/scripts/install-prism-qbit.sh b/.github/scripts/install-prism-qbit.sh new file mode 100755 index 000000000..fa1e44f9d --- /dev/null +++ b/.github/scripts/install-prism-qbit.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Install the pinned qbit release the PRISM native integration tests run +# against, checking its sha256 before anything is extracted. +# +# Usage: install-prism-qbit.sh +# +# The tarball is taken from the first of these that yields the pinned digest: +# 1. PRISM_QBIT_CACHE_DIR, when set (a cached copy whose digest mismatches is +# deleted, not used); +# 2. the release's browser download route on github.com; +# 3. the release-asset API route on api.github.com, which fronts the same +# storage through a different tier, so a github.com gateway error alone +# does not fail the install (no token: the repository is public). +# Each route is fetched with a bounded retry budget. A copy that verified is +# left in the cache directory so the next run needs no network at all. The +# script fails only when every source is unusable. +# +# The version, asset id and digest below are the pin; bump them together. The +# digest is the sha256 GitHub publishes for the asset +# (gh api repos/Qbit-Org/qbit/releases/assets/ --jq .digest). +set -euo pipefail + +destination="${1:?usage: install-prism-qbit.sh }" + +version=1.0.0 +asset_id=478569140 +asset="qbit-${version}-x86_64-linux-gnu.tar.gz" +sha256=ae121af03263b55d530e3f3e8719a71362d0950cba82f54a2bc2d6c437a029b5 + +release_url="https://github.com/Qbit-Org/qbit/releases/download/v${version}/${asset}" +asset_api_url="https://api.github.com/repos/Qbit-Org/qbit/releases/assets/${asset_id}" +retries=5 +retry_max_time=90 +cache_dir="${PRISM_QBIT_CACHE_DIR:-}" + +scratch="$(mktemp -d)" +trap 'rm -rf "${scratch}"' EXIT + +log() { + printf 'install-prism-qbit: %s\n' "$*" >&2 +} + +# verify : succeed when the file's sha256 is the pin; log both on mismatch. +verify() { + local actual + actual="$(sha256sum "$1" | cut -d ' ' -f 1)" + if [[ "${actual}" != "${sha256}" ]]; then + log "digest mismatch for $1: expected ${sha256}, got ${actual}" + return 1 + fi +} + +# fetch [curl args...]: download with a bounded +# retry budget and verify; on any failure remove the output and fail. +fetch() { + local route="$1" url="$2" output="$3" + shift 3 + log "fetching ${asset} from the ${route} (${url})" + if ! curl --fail --location --silent --show-error \ + --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + --retry "${retries}" --retry-max-time "${retry_max_time}" --retry-all-errors \ + "$@" --output "${output}" "${url}"; then + log "the ${route} did not deliver ${asset}" + rm -f "${output}" + return 1 + fi + if ! verify "${output}"; then + rm -f "${output}" + return 1 + fi +} + +tarball="" +if [[ -n "${cache_dir}" ]] && [[ -f "${cache_dir}/${asset}" ]]; then + if verify "${cache_dir}/${asset}"; then + log "using the cached ${asset} from ${cache_dir}" + tarball="${cache_dir}/${asset}" + else + log "discarding the cached ${asset} from ${cache_dir}" + rm -f "${cache_dir}/${asset}" + fi +fi + +if [[ -z "${tarball}" ]]; then + if fetch 'release download route' "${release_url}" "${scratch}/${asset}"; then + tarball="${scratch}/${asset}" + elif fetch 'release-asset API route' "${asset_api_url}" "${scratch}/${asset}" \ + --header 'Accept: application/octet-stream'; then + tarball="${scratch}/${asset}" + else + log "no source delivered ${asset} with digest ${sha256}: the release download route and the release-asset API route both failed" + exit 1 + fi + if [[ -n "${cache_dir}" ]]; then + mkdir -p "${cache_dir}" + cp "${tarball}" "${cache_dir}/${asset}.partial" + mv "${cache_dir}/${asset}.partial" "${cache_dir}/${asset}" + tarball="${cache_dir}/${asset}" + fi +fi + +mkdir -p "${destination}" +tar -xzf "${tarball}" -C "${destination}" +qbitd="${destination}/qbit-${version}/bin/qbitd" +if [[ ! -x "${qbitd}" ]]; then + log "${asset} did not contain an executable qbit-${version}/bin/qbitd" + exit 1 +fi +if [[ -n "${GITHUB_ENV:-}" ]]; then + printf 'QBITD_BIN=%s\n' "${qbitd}" >> "${GITHUB_ENV}" +fi +log "qbit ${version} -> ${qbitd}" diff --git a/.github/scripts/prism-load-run.sh b/.github/scripts/prism-load-run.sh new file mode 100755 index 000000000..193303457 --- /dev/null +++ b/.github/scripts/prism-load-run.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Run one checked-in load-harness preset and gate it (#521). The nightly +# workflow runs exactly this, and so can anyone reproducing a nightly result. +# +# Usage: prism-load-run.sh [operational flags] +# +# Any further arguments go to the harness after the preset's, so only flags +# the preset does not set are accepted: `--allow-dirty-tree` to reproduce a +# run from a modified checkout, say. A preset flag given again is refused. +# +# Expects release builds of qbit-prism-server, qbit-prism-load and +# qbit-prism-load-gate in ${TARGET_DIR:-target}/release, and PostgreSQL 16 +# server binaries in PG_BIN_DIR (default /usr/lib/postgresql/16/bin). +# +# Writes into the output directory: +# host.json the host fingerprint from scripts/prism_load_probe.py +# host: the runner label (RUNNER_LABEL, when set), +# CPU, memory, kernel, and the filesystem and block +# device under the cluster (mount options, write +# cache, FUA) +# pg_test_fsync.txt the WAL volume's commit cost, on the filesystem +# the harness builds its cluster on +# harness-exit-code the harness's exit code +# gate.md the gate's verdict table (a soak preset's soak +# gates included) +# gate-exit-code the gate's exit code: 0 or 1 is a verdict, anything +# else (2, a panic, a binary that would not start) is not +# everything the harness writes (load-harness-report.json, logs/, and for +# a soak preset soak-samples.jsonl, soak-events.jsonl, soak-report.md) +# +# Optional overrides of the preset's gates: PRISM_LOAD_MAX_SHORTFALL and +# PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS (empty keeps the preset's). The +# gate's table is also appended to GITHUB_STEP_SUMMARY when that is set. +# Exits with the gate's status: 0 pass, 1 fail, 2 unreadable inputs. +set -euo pipefail + +preset="${1:?usage: prism-load-run.sh [flags]}" +out="${2:?usage: prism-load-run.sh [flags]}" +shift 2 +root="$(git rev-parse --show-toplevel)" +target="${TARGET_DIR:-${root}/target}/release" +pg_bin="${PG_BIN_DIR:-/usr/lib/postgresql/16/bin}" +presets="${root}/crates/qbit-prism-load/presets" +# A deprecated name (presets/aliases.txt) runs the preset it was renamed to. +if [[ ! -f "${presets}/${preset}.json" && -f "${presets}/aliases.txt" ]]; then + renamed="$(awk -v old="${preset}" '$1 == old && $1 !~ /^#/ { print $2 }' "${presets}/aliases.txt")" + if [[ -n "${renamed}" ]]; then + echo "prism-load-run: ${preset} is deprecated; running ${renamed}" >&2 + preset="${renamed}" + fi +fi +preset_file="${presets}/${preset}.json" +test -f "${preset_file}" || { echo "no preset ${preset_file}" >&2; exit 2; } + +mkdir -p "${out}" +# The harness builds its cluster under TMPDIR, and PostgreSQL refuses a +# socket path over 107 bytes, so the directory is kept short. +export TMPDIR="${PRISM_LOAD_TMPDIR:-${RUNNER_TEMP:-/tmp}/pload}" +mkdir -p "${TMPDIR}" + +# #549: the fingerprint is evidence beside the result, so failing to take it +# is noted in the file and does not stop the run. +python3 "${root}/scripts/prism_load_probe.py" host --out "${out}/host.json" --dir "${TMPDIR}" \ + ${RUNNER_LABEL:+--runner "${RUNNER_LABEL}"} \ + || echo '{"error": "prism_load_probe.py host failed"}' > "${out}/host.json" + +{ + echo "# pg_test_fsync on the harness's cluster filesystem (${TMPDIR})" + df -h "${TMPDIR}" || true + "${pg_bin}/pg_test_fsync" -s 2 -f "${TMPDIR}/pg_test_fsync.out" +} > "${out}/pg_test_fsync.txt" 2>&1 || echo "pg_test_fsync failed; see above" >> "${out}/pg_test_fsync.txt" +rm -f "${TMPDIR}/pg_test_fsync.out" + +set +e +"${target}/qbit-prism-load" \ + --preset "${preset_file}" \ + --server-bin "${target}/qbit-prism-server" \ + --pg-bin-dir "${pg_bin}" \ + --out "${out}" \ + "$@" +code=$? +set -e +echo "${code}" > "${out}/harness-exit-code" + +gate=("${target}/qbit-prism-load-gate" + --report "${out}/load-harness-report.json" + --preset "${preset_file}" + --exit-code "${code}") +if [[ -n "${PRISM_LOAD_MAX_SHORTFALL:-}" ]]; then + gate+=(--max-shortfall "${PRISM_LOAD_MAX_SHORTFALL}") +fi +if [[ -n "${PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS:-}" ]]; then + gate+=(--tip-last-notify-p99-budget-ms "${PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS}") +fi +set +e +"${gate[@]}" > "${out}/gate.md" +status=$? +set -e +echo "${status}" > "${out}/gate-exit-code" +cat "${out}/gate.md" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + cat "${out}/gate.md" + echo + if [[ -f "${out}/soak-report.md" ]]; then + echo "
soak samples" + echo + sed -n '/^#### Samples/,$p' "${out}/soak-report.md" + echo + echo "
" + echo + fi + echo "
Host" + echo + echo '```json' + cat "${out}/host.json" + echo '```' + echo + echo "
" + echo + echo "
pg_test_fsync" + echo + echo '```' + cat "${out}/pg_test_fsync.txt" + echo '```' + echo + echo "
" + echo + } >> "${GITHUB_STEP_SUMMARY}" +fi +exit "${status}" diff --git a/.github/scripts/prism-stratum-fuzz.sh b/.github/scripts/prism-stratum-fuzz.sh new file mode 100755 index 000000000..7f6accf86 --- /dev/null +++ b/.github/scripts/prism-stratum-fuzz.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Run the Stratum cargo-fuzz targets for a fixed time and gate on no crash +# (#575). The nightly workflow runs exactly this, and so can anyone +# reproducing a nightly result. +# +# Usage: prism-stratum-fuzz.sh [seconds] +# +# Installs the pinned nightly toolchain and the pinned cargo-fuzz release +# (checked against its sha256), builds the targets in +# crates/qbit-prism-server/fuzz, and runs all three at once for the given +# seconds each (default 1200): stratum_session, stratum_lines and +# codec_parsers. Each starts from the checked-in seeds and +# /, which it grows; afterwards each target's corpus +# is merged down to the inputs that add coverage, so a cached corpus stays +# small from night to night. +# +# Writes into the output directory: +# .log libFuzzer's log, final stats included +# artifacts// crash-*, leak-*, timeout-* and oom-* inputs +# summary.md one row per target, also appended to +# GITHUB_STEP_SUMMARY when that is set +# Exits 1 when any target found a crash, leak, timeout or out-of-memory +# input or exited abnormally, after every target has run; 0 otherwise. +# Reproduce an artifact with: +# cargo + fuzz run --fuzz-dir crates/qbit-prism-server/fuzz +# and PRISM_FUZZ_TRACE=1 to print the conversation. +set -euo pipefail + +corpus="${1:?usage: prism-stratum-fuzz.sh [seconds]}" +out="${2:?usage: prism-stratum-fuzz.sh [seconds]}" +seconds="${3:-1200}" +[[ "${seconds}" =~ ^[1-9][0-9]*$ ]] || { echo "prism-stratum-fuzz: seconds must be a positive integer" >&2; exit 2; } + +# The toolchain the targets were last run and measured on. cargo-fuzz needs +# nightly for its sanitizer flags; the workspace itself stays on the stable +# pin in rust-toolchain.toml. +NIGHTLY=nightly-2026-09-24 +CARGO_FUZZ_VERSION=0.13.2 +CARGO_FUZZ_SHA256=b5b704018b63e0f151c17a057ac53b5111e1db545d1b9f72fee79f08a545931c +TARGETS=(stratum_session stratum_lines codec_parsers) +declare -A MAX_LEN=([stratum_session]=16384 [stratum_lines]=20000 [codec_parsers]=8192) + +root="$(git rev-parse --show-toplevel)" +fuzz_dir="${root}/crates/qbit-prism-server/fuzz" +mkdir -p "${corpus}" "${out}/artifacts" +corpus="$(cd -- "${corpus}" && pwd)" +out="$(cd -- "${out}" && pwd)" +tools="$(mktemp -d)" +trap 'rm -rf "${tools}"' EXIT + +rustup toolchain install "${NIGHTLY}" --profile minimal --no-self-update + +asset="cargo-fuzz-${CARGO_FUZZ_VERSION}-x86_64-unknown-linux-musl.tar.gz" +curl --fail --location --silent --show-error --retry 12 --retry-delay 15 \ + --retry-max-time 180 --connect-timeout 10 --max-time 60 \ + --output "${tools}/${asset}" \ + "https://github.com/rust-fuzz/cargo-fuzz/releases/download/${CARGO_FUZZ_VERSION}/${asset}" +printf '%s %s\n' "${CARGO_FUZZ_SHA256}" "${tools}/${asset}" | sha256sum --check --quiet +tar -xzf "${tools}/${asset}" -C "${tools}" cargo-fuzz +export PATH="${tools}:${PATH}" +cargo fuzz --version + +cd "${fuzz_dir}" +# The release binary is a musl build, whose default target would be musl too. +cargo "+${NIGHTLY}" fuzz build -O --target x86_64-unknown-linux-gnu +bin_dir="${CARGO_TARGET_DIR:-${fuzz_dir}/target}/x86_64-unknown-linux-gnu/release" + +pids=() +for target in "${TARGETS[@]}"; do + mkdir -p "${corpus}/${target}" "${out}/artifacts/${target}" + "${bin_dir}/${target}" "${corpus}/${target}" "${fuzz_dir}/seeds/${target}" \ + -dict="${fuzz_dir}/stratum.dict" -max_len="${MAX_LEN[${target}]}" \ + -max_total_time="${seconds}" -rss_limit_mb=2048 -malloc_limit_mb=512 \ + -timeout=30 -print_final_stats=1 \ + -artifact_prefix="${out}/artifacts/${target}/" \ + > "${out}/${target}.log" 2>&1 & + pids+=("$!") +done + +status=0 +declare -A EXIT MERGE +for i in "${!TARGETS[@]}"; do + code=0 + wait "${pids[${i}]}" || code=$? + EXIT[${TARGETS[${i}]}]="${code}" + (( code == 0 )) || status=1 +done + +{ + echo "### Stratum fuzzing, ${seconds} s per target on ${NIGHTLY}" + echo + echo "| target | exit | runs | exec/s | coverage (edges) | features | corpus | findings |" + echo "| --- | --- | --- | --- | --- | --- | --- | --- |" +} > "${out}/summary.md" +for target in "${TARGETS[@]}"; do + log="${out}/${target}.log" + final_stat() { sed -n "s/^stat::$1: *//p" "${log}" | tail -n 1; } + last="$(grep -E '^#[0-9]+' "${log}" | tail -n 1 || true)" + cov="$(sed -nE 's/.* cov: ([0-9]+).*/\1/p' <<< "${last}")" + ft="$(sed -nE 's/.* ft: ([0-9]+).*/\1/p' <<< "${last}")" + findings="$(find "${out}/artifacts/${target}" -type f | wc -l)" + # Keep only inputs that add coverage, so the cache does not grow nightly. + # A merge that fails keeps the whole grown corpus instead of a partial one. + merged="$(mktemp -d)" + MERGE[${target}]=0 + "${bin_dir}/${target}" -merge=1 -max_len="${MAX_LEN[${target}]}" -rss_limit_mb=2048 \ + -malloc_limit_mb=512 -timeout=30 -artifact_prefix="${out}/artifacts/${target}/merge-" \ + "${merged}" "${corpus}/${target}" > "${out}/${target}.merge.log" 2>&1 || MERGE[${target}]=$? + if [[ "${MERGE[${target}]}" == 0 ]]; then + rm -rf "${corpus:?}/${target}" + mv "${merged}" "${corpus}/${target}" + else + status=1 + rm -rf "${merged}" + fi + size="$(find "${corpus}/${target}" -type f | wc -l)" + echo "| ${target} | ${EXIT[${target}]} | $(final_stat number_of_executed_units) | $(final_stat average_exec_per_sec) | ${cov:-?} | ${ft:-?} | ${size} | ${findings} |" \ + >> "${out}/summary.md" +done +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "${out}/summary.md" >> "${GITHUB_STEP_SUMMARY}" +fi +cat "${out}/summary.md" +for target in "${TARGETS[@]}"; do + if [[ "${EXIT[${target}]}" != 0 ]]; then + echo "::error::${target} failed; its input is under artifacts/${target} and its log is ${target}.log" + grep -m 5 -E 'invariant violated|panicked at|SUMMARY|ERROR' "${out}/${target}.log" || true + fi + if [[ "${MERGE[${target}]}" != 0 ]]; then + echo "::error::${target}'s corpus merge failed (exit ${MERGE[${target}]}); its corpus was kept unmerged, see ${target}.merge.log and artifacts/${target}/merge-*" + grep -m 5 -E 'invariant violated|panicked at|SUMMARY|ERROR' "${out}/${target}.merge.log" || true + fi +done +exit "${status}" diff --git a/.github/scripts/test-install-pinned-tool.sh b/.github/scripts/test-install-pinned-tool.sh new file mode 100644 index 000000000..10d09df0e --- /dev/null +++ b/.github/scripts/test-install-pinned-tool.sh @@ -0,0 +1,834 @@ +#!/usr/bin/env bash +# Prove that install-pinned-tool.sh rides out a release-host outage, installs +# from its verified cache with the network gone, and refuses anything that does +# not match the pinned sha256 on the download path and the cache path alike, +# without reaching the network, Docker or GitHub. +# +# A Python http.server on 127.0.0.1 stands in for the release host. Each +# scenario sets its behaviour (serve, 504 until a deadline, 404, stall, refuse) +# through a control file, and the server logs every connection so the test can +# assert request counts, not only exit statuses. +# +# The installer hardcodes its pins and its 180/210-second timing, so a hermetic +# test can neither serve bytes with the published digests nor wait out the real +# windows. Two narrow bridges close that gap and leave every guard as the +# committed code: +# * A routing-only curl shim on PATH rewrites the pinned https://github.com/ +# URL to the loopback server and execs the real curl with every other +# argument untouched, so the installer's own retry, deadline and error +# handling run as written. +# * Scripts under test are derived from the committed installer by replacing +# exactly four whole lines: the two sha256= pins (with the digests of the +# fixture assets, computed here with sha256sum) and the two timing lines +# (timeout --signal=KILL and --retry ...). Each line must match exactly +# once, and a diff against the committed file must show only those lines, +# so an edit to any guard flows straight into the test and an edit to a +# substituted line fails loudly. sha256sum is never shimmed: verify() runs +# as committed, only the expected value differs. Two variants are derived: +# a 15-second deadline for the outage scenarios and a 2-second deadline so +# the stalled-transfer scenario finishes fast. +# The pure-failure scenarios (404, substituted bytes, poisoned cache, non-regular +# cache) also run against the unmodified committed installer, with its real pins +# and timing, through the shim alone. +# +# Bad assets and poisoned cache entries carry an execution marker: running them +# touches a sentinel file, which must never appear. One warm-cache run uses a +# cache directory whose path contains a newline: sha256sum --check cannot name +# such a file, so it passes only if the private copy, not the shared entry, is +# what is checked. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +INSTALLER="${SCRIPT_DIR}/install-pinned-tool.sh" +TOOLS=(cargo-deny hadolint) + +# The pins as they must stay; a quiet bump trips this test. +EXPECTED_CARGO_DENY_VERSION=0.20.2 +EXPECTED_CARGO_DENY_SHA256=9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f +EXPECTED_HADOLINT_VERSION=2.15.1 +EXPECTED_HADOLINT_SHA256=c7187db94eeeeca956519a6af171adc31453941a1e777961f6e680f697c8c507 + +# Derived-variant timing. Retries run every second; the outage lasts longer +# than the seven seconds the pre-#380 installer retried for. +OUTAGE_SECONDS=8 +OUTAGE_DEADLINE=15 +STALL_DEADLINE=2 +RETRY_MAX_TIME=12 +# Bound on one installer run, so a regression that hangs (a cp blocking on a +# FIFO, a 404 that retries) fails this test instead of the job. +RUN_LIMIT=20 + +if [[ -z "${EPOCHREALTIME:-}" ]]; then + echo 'pinned-tool test: FAIL: bash 5 or newer is required (EPOCHREALTIME)' >&2 + exit 1 +fi +START_US="${EPOCHREALTIME/./}" +scenarios=0 +assertions=0 +SCENARIO="" +WORK="" +SERVER_PID="" +RUN_OUTPUT="" +REQUEST_LOG="" + +fail() { + printf 'pinned-tool test: FAIL: %s\n' "$*" >&2 + if [[ -n "${SCENARIO}" ]]; then + printf 'pinned-tool test: while running %s\n' "${SCENARIO}" >&2 + fi + if [[ -n "${RUN_OUTPUT}" && -s "${RUN_OUTPUT}" ]]; then + printf -- '--- installer output ---\n' >&2 + cat "${RUN_OUTPUT}" >&2 + fi + if [[ -n "${REQUEST_LOG}" && -s "${REQUEST_LOG}" ]]; then + printf -- '--- release-host requests ---\n' >&2 + cat "${REQUEST_LOG}" >&2 + fi + exit 1 +} + +cleanup() { + if [[ -n "${SERVER_PID}" ]]; then + kill -TERM "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi + if [[ -n "${WORK}" ]]; then + rm -rf -- "${WORK}" + fi +} +trap cleanup EXIT + +assert_eq() { # + assertions=$((assertions + 1)) + [[ "$1" == "$2" ]] || fail "$3: expected '$2', got '$1'" +} + +assert_ne() { # + assertions=$((assertions + 1)) + [[ "$1" != "$2" ]] || fail "$3: got '$1'" +} + +assert_ge() { # + assertions=$((assertions + 1)) + (( $1 >= $2 )) || fail "$3: expected at least $2, got $1" +} + +assert_lt() { # + assertions=$((assertions + 1)) + (( $1 < $2 )) || fail "$3: expected under $2, got $1" +} + +assert_prefix() { # + assertions=$((assertions + 1)) + [[ "$1" == "$2"* ]] || fail "$3: '$1' does not start with '$2'" +} + +assert_exists() { # + assertions=$((assertions + 1)) + [[ -e "$1" || -L "$1" ]] || fail "$2: $1 is missing" +} + +assert_absent() { # + assertions=$((assertions + 1)) + [[ ! -e "$1" && ! -L "$1" ]] || fail "$2: $1 exists" +} + +assert_output_contains() { # + assertions=$((assertions + 1)) + grep -qF -- "$1" "${RUN_OUTPUT}" || fail "installer output lacks '$1'" +} + +assert_output_lacks() { # + assertions=$((assertions + 1)) + ! grep -qF -- "$1" "${RUN_OUTPUT}" || fail "installer output contains '$1': $2" +} + +assert_same_bytes() { # + assertions=$((assertions + 1)) + cmp -s -- "$1" "$2" || fail "$3: $1 differs from $2" +} + +scenario() { #