From e824ab158420bb74ffddcb5e7ec61b2949c09fc4 Mon Sep 17 00:00:00 2001 From: Sanket Kanjalkar Date: Wed, 5 Aug 2026 11:03:18 -0700 Subject: [PATCH 1/2] Bound the block target in calculateEstimates and the constructor Simulation cost grows with the block target: getExpectedBlocksMined evaluates a Poisson tail of 4 * target entries and then simulates that many blocks, so a target of 1_000_000 spends roughly two minutes of CPU. Neither the numOfBlocks argument nor the blockTargets constructor list had an upper bound, and bitcoin-augur-server passes an HTTP query parameter straight into numOfBlocks, so one unauthenticated request can saturate a core. Reject targets above MAX_BLOCK_TARGET, one week of blocks. --- lib/api/lib.api | 1 + .../kotlin/xyz/block/augur/FeeEstimator.kt | 16 +++++++++++ .../xyz/block/augur/FeeEstimatorTest.kt | 28 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/lib/api/lib.api b/lib/api/lib.api index ba6425f..f5aede3 100644 --- a/lib/api/lib.api +++ b/lib/api/lib.api @@ -32,6 +32,7 @@ public final class xyz/block/augur/FeeEstimate { public final class xyz/block/augur/FeeEstimator { public static final field Companion Lxyz/block/augur/FeeEstimator$Companion; + public static final field MAX_BLOCK_TARGET D public fun ()V public fun (Ljava/util/List;)V public fun (Ljava/util/List;Ljava/util/List;)V diff --git a/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt b/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt index 64f1620..c250f0a 100644 --- a/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt +++ b/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt @@ -72,6 +72,10 @@ public class FeeEstimator @JvmOverloads public constructor( require(blockTargets.isNotEmpty()) { "At least one block target must be provided" } require(probabilities.all { it in 0.0..1.0 }) { "All probabilities must be between 0.0 and 1.0" } require(blockTargets.all { it > 0 }) { "All block targets must be positive" } + // Same cost bound as numOfBlocks below: a target of a million blocks simulates a million blocks. + require(blockTargets.all { it <= MAX_BLOCK_TARGET }) { + "All block targets must be at most $MAX_BLOCK_TARGET, was ${blockTargets.filter { it > MAX_BLOCK_TARGET }}" + } require(maxFeeRate > 0.0) { "maxFeeRate must be positive, was $maxFeeRate" } bucketLayout = BucketLayout(minFeeRate) feeEstimatesCalculator = FeeEstimatesCalculator(probabilities, blockTargets, bucketLayout, maxFeeRate) @@ -93,6 +97,12 @@ public class FeeEstimator @JvmOverloads public constructor( // If numOfBlocks is specified then it needs to be at least 3, // since we can't simulate partial blocks being mined require(numOfBlocks == null || numOfBlocks >= 3.0) { "numOfBlocks must be at least 3 if specified" } + // It also needs an upper bound: getExpectedBlocksMined evaluates a Poisson tail of + // 4 * numOfBlocks entries and then simulates that many blocks, so an unbounded value costs + // minutes of CPU per call -- and callers behind an HTTP service pass a request parameter through. + require(numOfBlocks == null || numOfBlocks <= MAX_BLOCK_TARGET) { + "numOfBlocks must be at most $MAX_BLOCK_TARGET if specified, was $numOfBlocks" + } if (mempoolSnapshots.isEmpty()) { return FeeEstimate(emptyMap(), Instant.now()) @@ -199,5 +209,11 @@ public class FeeEstimator @JvmOverloads public constructor( * simulation ceiling (bucket 1000) pass the filter. */ public val DEFAULT_MAX_FEE_RATE: Double = FeeEstimatesCalculator.DEFAULT_MAX_FEE_RATE + + /** + * Largest supported block target, one week of blocks. Simulation cost grows linearly with the + * target, so this bound keeps a caller-supplied `numOfBlocks` from costing minutes of CPU. + */ + public const val MAX_BLOCK_TARGET: Double = 1008.0 } } diff --git a/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt index 4c0b259..9fc3974 100644 --- a/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt +++ b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt @@ -329,6 +329,34 @@ class FeeEstimatorTest { } } + @Test + fun `test calculateEstimates throws if numOfBlocks exceeds the maximum block target`() { + // Simulation cost grows with the target, so an unbounded value burns minutes of CPU in a single + // call -- and callers behind an HTTP service pass a request parameter straight through. + val snapshots = TestUtils.createSnapshotSequence(blockCount = 5, snapshotsPerBlock = 3) + assertFailsWith { + feeEstimator.calculateEstimates(snapshots, numOfBlocks = 1_000_000.0) + } + assertFailsWith { + feeEstimator.calculateEstimates(snapshots, numOfBlocks = FeeEstimator.MAX_BLOCK_TARGET + 1) + } + // The bound itself is still accepted. + feeEstimator.calculateEstimates(snapshots, numOfBlocks = FeeEstimator.MAX_BLOCK_TARGET) + } + + @Test + fun `test constructor throws if a block target exceeds the maximum`() { + // blockTargets reaches the same simulation cost as numOfBlocks, so it needs the same bound. + assertFailsWith { + FeeEstimator(blockTargets = listOf(3.0, 1_000_000.0)) + } + assertFailsWith { + feeEstimator.configure(blockTargets = listOf(FeeEstimator.MAX_BLOCK_TARGET + 1)) + } + // The bound itself is still accepted. + FeeEstimator(blockTargets = listOf(FeeEstimator.MAX_BLOCK_TARGET)) + } + @Test fun `test constructor throws if minFeeRate is zero or negative`() { assertFailsWith { From 3a0a78961ad8a04dc9cdeee7db304676e410f943 Mon Sep 17 00:00:00 2001 From: Sanket Kanjalkar Date: Wed, 5 Aug 2026 11:01:46 -0700 Subject: [PATCH 2/2] Report null instead of a wrong fee rate when an estimate is unavailable 1 sat/vB meant three different things: a real recommendation, "no data", and "the simulation failed". Callers could not tell them apart, so an unanswerable request came back as the cheapest possible fee. - A confidence level above 1 - exp(-blockTarget) cannot be met at any fee rate. runSimulations returned bucket 0 for it, which is a real fee rate. - findBestIndex flagged failure with bucketMax + 1, whose fee rate is 22247.84 sat/vB. That only stayed hidden because it sits above the default maxFeeRate; raising maxFeeRate surfaced it as an estimate. - enforceMonotonicity clamped each target to the previous one, so a collapsed short target dragged every longer target in the column down with it. - The short/long blend ramp 1 - (1 - t/144)^2 turns back down past the window and reaches -35 at 1008 blocks, extrapolating the two estimates apart instead of averaging them. It also ignored longTermWindowDuration. - InflowCalculator divided by a zero span whenever no block height had two snapshots, the normal state for a per-block collector: Infinity, then NaN in every bucket, then a table of nulls with no indication why. It also measured the span in whole seconds, truncating sub-second spacing to zero. - Unsorted blockTargets broke the monotonicity walk. - mineBlock treated a negative bucket weight as freed capacity. - getNearestBlockTarget resolved exact ties by map iteration order and could overflow on extreme inputs. Fixture weights are now seeded so the new assertions are reproducible. --- .../kotlin/xyz/block/augur/FeeEstimate.kt | 10 +- .../kotlin/xyz/block/augur/FeeEstimator.kt | 41 +++-- .../augur/internal/FeeEstimatesCalculator.kt | 57 +++++-- .../block/augur/internal/InflowCalculator.kt | 19 ++- .../block/augur/FeeEstimatorContractTest.kt | 149 ++++++++++++++++++ .../xyz/block/augur/FeeEstimatorTest.kt | 43 +++-- .../internal/FeeEstimatesCalculatorTest.kt | 93 ++++++++++- .../augur/internal/InflowCalculatorTest.kt | 50 ++++++ .../kotlin/xyz/block/augur/test/TestUtils.kt | 23 ++- 9 files changed, 434 insertions(+), 51 deletions(-) create mode 100644 lib/src/test/kotlin/xyz/block/augur/FeeEstimatorContractTest.kt diff --git a/lib/src/main/kotlin/xyz/block/augur/FeeEstimate.kt b/lib/src/main/kotlin/xyz/block/augur/FeeEstimate.kt index 517459c..52384cc 100644 --- a/lib/src/main/kotlin/xyz/block/augur/FeeEstimate.kt +++ b/lib/src/main/kotlin/xyz/block/augur/FeeEstimate.kt @@ -69,6 +69,8 @@ public data class FeeEstimate( * * This is useful when the exact requested block target is not available. * + * Ties are broken toward the smaller target, which carries a fee rate at least as high. + * * @param targetBlocks The desired confirmation target in blocks * @return The nearest available block target, or null if no estimates are available */ @@ -76,8 +78,12 @@ public data class FeeEstimate( if (estimates.isEmpty()) return null if (estimates.containsKey(targetBlocks)) return targetBlocks - return estimates.keys - .minByOrNull { kotlin.math.abs(it - targetBlocks) } + // Comparing in Long avoids overflow on extreme inputs, and the second comparator makes the + // result independent of map iteration order -- previously an exact tie returned whichever key + // the map happened to yield first. + return estimates.keys.minWithOrNull( + compareBy({ kotlin.math.abs(it.toLong() - targetBlocks) }, { it }), + ) } /** diff --git a/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt b/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt index c250f0a..49b7650 100644 --- a/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt +++ b/lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt @@ -41,8 +41,11 @@ import java.time.Instant * val feeRate = estimate.getFeeRate(targetBlocks = 6, probability = 0.95) * ``` * - * @property probabilities The confidence levels to calculate (default: 5%, 20%, 50%, 80%, 95%) - * @property blockTargets The block confirmation targets to estimate for (default: 3, 6, 9, 12, 18, 24, 36, 48, 72, 96, 144) + * @property probabilities The confidence levels to calculate (default: 5%, 20%, 50%, 80%, 95%). + * A level above `1 - exp(-blockTarget)` — the chance any block is found within the target, 0.9502 + * at 3 blocks — is unanswerable at any fee rate and estimates null for that cell. + * @property blockTargets The block confirmation targets to estimate for (default: 3, 6, 9, 12, 18, + * 24, 36, 48, 72, 96, 144). Sorted internally. * @property minFeeRate The minimum fee rate in sat/vB for the simulation lower bound (default: 1.0). * Set to 0.1 for Bitcoin Core 29.1/30.0+ nodes that support sub-1 sat/vB fee rates. Snapshots * store all bucketed transactions regardless of this value; `minFeeRate` controls which buckets @@ -58,29 +61,42 @@ import java.time.Instant */ public class FeeEstimator @JvmOverloads public constructor( private val probabilities: List = DEFAULT_PROBABILITIES, - private val blockTargets: List = DEFAULT_BLOCK_TARGETS, + blockTargets: List = DEFAULT_BLOCK_TARGETS, private val shortTermWindowDuration: Duration = Duration.ofMinutes(30), private val longTermWindowDuration: Duration = Duration.ofHours(24), private val minFeeRate: Double = DEFAULT_MIN_FEE_RATE, private val maxFeeRate: Double = DEFAULT_MAX_FEE_RATE, ) { + /** + * Sorted ascending. [FeeEstimatesCalculator.enforceMonotonicity] walks block targets in order, so + * an unsorted list used to clamp every short target down to the longest target's fee rate. + */ + private val blockTargets: List = blockTargets.sorted() + + /** The long-term window in blocks, where the short/long blend ramp reaches a pure long-term estimate. */ + private val longTermWindowBlocks: Double = + longTermWindowDuration.toMillis() / (FeeEstimatesCalculator.MINUTES_PER_BLOCK * 60_000.0) + private val bucketLayout: BucketLayout private val feeEstimatesCalculator: FeeEstimatesCalculator init { require(probabilities.isNotEmpty()) { "At least one probability level must be provided" } - require(blockTargets.isNotEmpty()) { "At least one block target must be provided" } + require(this.blockTargets.isNotEmpty()) { "At least one block target must be provided" } require(probabilities.all { it in 0.0..1.0 }) { "All probabilities must be between 0.0 and 1.0" } - require(blockTargets.all { it > 0 }) { "All block targets must be positive" } + require(this.blockTargets.all { it > 0 }) { "All block targets must be positive" } // Same cost bound as numOfBlocks below: a target of a million blocks simulates a million blocks. - require(blockTargets.all { it <= MAX_BLOCK_TARGET }) { - "All block targets must be at most $MAX_BLOCK_TARGET, was ${blockTargets.filter { it > MAX_BLOCK_TARGET }}" + require(this.blockTargets.all { it <= MAX_BLOCK_TARGET }) { + "All block targets must be at most $MAX_BLOCK_TARGET, was ${this.blockTargets.filter { it > MAX_BLOCK_TARGET }}" } require(maxFeeRate > 0.0) { "maxFeeRate must be positive, was $maxFeeRate" } bucketLayout = BucketLayout(minFeeRate) - feeEstimatesCalculator = FeeEstimatesCalculator(probabilities, blockTargets, bucketLayout, maxFeeRate) + feeEstimatesCalculator = newCalculator(this.blockTargets) } + private fun newCalculator(targets: List): FeeEstimatesCalculator = + FeeEstimatesCalculator(probabilities, targets, bucketLayout, maxFeeRate, longTermWindowBlocks) + /** * Calculates fee estimates based on historical mempool snapshots. * @@ -112,13 +128,20 @@ public class FeeEstimator @JvmOverloads public constructor( val orderedSnapshots = mempoolSnapshots.sortedBy { it.timestamp } val simdSnapshots = orderedSnapshots.map { MempoolSnapshotF64Array.fromMempoolSnapshot(it, bucketLayout) } + // Inflow is only observable between two snapshots at the same block height, so without such a + // pair there is no estimate to make. Previously this divided by a zero span and returned a table + // of nulls instead of saying so. + if (simdSnapshots.groupingBy { it.blockHeight }.eachCount().none { it.value > 1 }) { + return FeeEstimate(emptyMap(), orderedSnapshots.last().timestamp) + } + // Extract latest mempool weights and calculate inflow rates val latestMempoolWeights = simdSnapshots.last().buckets val shortTermInflows = InflowCalculator.calculateInflows(simdSnapshots, shortTermWindowDuration, bucketLayout) val longTermInflows = InflowCalculator.calculateInflows(simdSnapshots, longTermWindowDuration, bucketLayout) val (calculator, targets) = if (numOfBlocks != null) { - FeeEstimatesCalculator(probabilities, listOf(numOfBlocks), bucketLayout, maxFeeRate) to listOf(numOfBlocks) + newCalculator(listOf(numOfBlocks)) to listOf(numOfBlocks) } else { feeEstimatesCalculator to blockTargets } diff --git a/lib/src/main/kotlin/xyz/block/augur/internal/FeeEstimatesCalculator.kt b/lib/src/main/kotlin/xyz/block/augur/internal/FeeEstimatesCalculator.kt index 7118e85..438e95e 100644 --- a/lib/src/main/kotlin/xyz/block/augur/internal/FeeEstimatesCalculator.kt +++ b/lib/src/main/kotlin/xyz/block/augur/internal/FeeEstimatesCalculator.kt @@ -33,6 +33,7 @@ internal class FeeEstimatesCalculator( private val blockTargets: List, private val bucketLayout: BucketLayout = BucketLayout.DEFAULT, private val maxFeeRate: Double = DEFAULT_MAX_FEE_RATE, + private val longTermWindowBlocks: Double = DEFAULT_LONG_TERM_WINDOW_BLOCKS, ) { private val expectedBlocksMined by lazy { getExpectedBlocksMined() } @@ -43,7 +44,8 @@ internal class FeeEstimatesCalculator( * @param shortIntervalInflows Short-term inflow data (typically 30 minutes) * @param longIntervalInflows Long-term inflow data (typically 24 hours) * @return A 2D array of fee estimates where each element corresponds to a specific - * block target and probability level. Values exceeding [maxFeeRate] are null. + * block target and probability level. An element is null when it exceeds [maxFeeRate], + * or when no fee rate can satisfy that (block target, probability) pair at all. */ fun getFeeEstimates( mempoolSnapshot: F64Array, @@ -96,13 +98,16 @@ internal class FeeEstimatesCalculator( probabilities.indices.forEach { probIndex -> val expectedBlocks = expectedBlocksMined[blockTargetIndex, probIndex].toInt() - // Run individual simulation and store result + // Run individual simulation and store result. NaN marks "no fee rate satisfies this pair" + // and prepareResultArray turns it into null. Bucket 0 must never stand in for failure: it is + // a real fee rate (minFeeRate, 1 sat/vB by default), so a caller could not tell a genuine + // 1 sat/vB recommendation apart from a failed simulation. result[blockTargetIndex, probIndex] = runSimulation( initialWeights, addedWeights, expectedBlocks, meanBlocks, - )?.toDouble() ?: 0.0 + )?.toDouble() ?: Double.NaN } } @@ -111,7 +116,9 @@ internal class FeeEstimatesCalculator( /** * Simulates mining blocks and returns the weight index corresponding to the - * lowest fee rate that would result in the transaction getting mined. + * lowest fee rate that would result in the transaction getting mined, or null when no + * simulation is possible (no blocks are expected to be mined at this confidence level) or + * when even the highest fee rate bucket would not clear. */ internal fun runSimulation( initialWeights: F64Array, @@ -154,7 +161,10 @@ internal class FeeEstimatesCalculator( var weightUnitsRemaining = blockSize for (i in 0 until weightsRemaining.length) { - val removedWeight = min(weightsRemaining[i], weightUnitsRemaining) + // coerceAtLeast(0.0) stops a negative bucket weight from *adding* to the block's remaining + // capacity, which would otherwise let a single block mine more than blockSize weight units + // and make the whole mempool look clearable at the minimum fee rate. + val removedWeight = min(weightsRemaining[i], weightUnitsRemaining).coerceAtLeast(0.0) weightUnitsRemaining -= removedWeight weightsRemaining[i] -= removedWeight } @@ -162,9 +172,9 @@ internal class FeeEstimatesCalculator( } /** - * Find the index of the last bucket that is fully mined. + * Find the index of the last bucket that is fully mined, or null if no bucket is. */ - internal fun findBestIndex(weightsRemaining: F64Array): Int { + internal fun findBestIndex(weightsRemaining: F64Array): Int? { // The last mined bucket will occur just before the first non-zero remaining weight. val index = weightsRemaining.toDoubleArray().indexOfFirst { it != 0.0 } - 1 @@ -173,7 +183,7 @@ internal class FeeEstimatesCalculator( // Else, createFeeRateBuckets reversed the order, so subtract to recover the original index. return when (index) { -2 -> bucketLayout.bucketMin // all weights are zero so we can use the cheapest fee rate - -1 -> bucketLayout.bucketMax + 1 // return null + -1 -> null // not even the highest fee rate bucket cleared, so no answer exists else -> bucketLayout.toBucketIndex(index) } } @@ -185,9 +195,13 @@ internal class FeeEstimatesCalculator( shortEstimates: F64Array, longEstimates: F64Array, ): F64Array { - // The longer estimates are weighted more heavily for longer intervals. For example, the - // weighted estimate for 24 hours (144 blocks) is exactly equal to the longEstimate. - val weights = blockTargets.map { 1 - (1 - it / 144.0).pow(2) } + // The longer estimates are weighted more heavily for longer intervals, reaching a pure + // long-term estimate at the long-term window (144 blocks for the default 24 hours). + // + // coerceAtMost(1.0) saturates the ramp past that point. Without it the parabola turns back down, + // returning to 0 at twice the window and reaching -35 at 1008 blocks, which extrapolates the two + // estimates apart instead of averaging them. + val weights = blockTargets.map { 1 - (1 - (it / longTermWindowBlocks).coerceAtMost(1.0)).pow(2) } val weightedEstimates = F64Array(shortEstimates.shape[0], shortEstimates.shape[1]) for (i in 0 until weightedEstimates.shape[0]) { @@ -203,13 +217,18 @@ internal class FeeEstimatesCalculator( internal fun convertBucketsToFeeRates(bucketEstimates: F64Array): F64Array = (bucketEstimates / 100.0).exp() /** - * Converts fee estimates to the final nullable array format and filters fees above the maximum bucket's fee rate. + * Converts fee estimates to the final nullable array format, dropping cells that no fee rate can + * satisfy along with any fee above [maxFeeRate]. * F64Array can't accommodate nulls so we convert to traditional arrays. */ private fun prepareResultArray(feeRates: F64Array): Array> { return Array(feeRates.shape[0]) { blockTargetIndex -> Array(feeRates.shape[1]) { probabilityIndex -> - feeRates[blockTargetIndex, probabilityIndex].takeIf { it <= maxFeeRate } + // isFinite() drops the NaN markers runSimulations writes for unanswerable cells, and stays + // independent of maxFeeRate. The old out-of-band bucket sentinel did not: it relied on the + // gap between exp(1001/100) and DEFAULT_MAX_FEE_RATE, so raising maxFeeRate surfaced + // 22247.84 sat/vB as a real estimate. + feeRates[blockTargetIndex, probabilityIndex].takeIf { it.isFinite() && it <= maxFeeRate } } } } @@ -241,12 +260,18 @@ internal class FeeEstimatesCalculator( * Ensures that fee rates decrease (or stay the same) as block targets increase. * For each probability, if a fee rate is higher than the previous one, * it is set equal to the previous rate. + * + * Assumes [blockTargets] is in ascending order; [xyz.block.augur.FeeEstimator] sorts it. */ internal fun enforceMonotonicity(feeRates: F64Array): F64Array { val result = feeRates.copy() for (j in 0 until result.shape[1]) { var prevRate = Double.POSITIVE_INFINITY for (i in 0 until result.shape[0]) { + // Skip unavailable cells, and in particular don't let one become the running bound: a + // single unanswerable short target would otherwise clamp every longer target in the + // column down to it, discarding estimates that were computed correctly. + if (result[i, j].isNaN()) continue if (result[i, j] > prevRate) { result[i, j] = prevRate } @@ -259,6 +284,12 @@ internal class FeeEstimatesCalculator( companion object { const val BLOCK_SIZE_WEIGHT_UNITS = 4_000_000 + /** Bitcoin's target block interval, used to convert window durations into block counts. */ + const val MINUTES_PER_BLOCK = 10.0 + + /** Blocks in the default 24 hour long-term window: 24 * 60 / 10. */ + const val DEFAULT_LONG_TERM_WINDOW_BLOCKS = 144.0 + // Rounded up from exp(10) ≈ 22026.47 so estimates at the simulation ceiling pass the <= filter const val DEFAULT_MAX_FEE_RATE = 22027.0 } diff --git a/lib/src/main/kotlin/xyz/block/augur/internal/InflowCalculator.kt b/lib/src/main/kotlin/xyz/block/augur/internal/InflowCalculator.kt index 9523d38..4e3848d 100644 --- a/lib/src/main/kotlin/xyz/block/augur/internal/InflowCalculator.kt +++ b/lib/src/main/kotlin/xyz/block/augur/internal/InflowCalculator.kt @@ -26,12 +26,16 @@ import java.time.Duration * during the time period being estimated. */ internal object InflowCalculator { + private val TEN_MINUTES: Duration = Duration.ofMinutes(10) + /** * Calculates inflow rates based on historical snapshots. * * @param mempoolSnapshots List of mempool snapshots * @param timeframe Duration to consider for inflow calculation - * @return Array of inflow rates by fee rate bucket + * @return Array of inflow rates by fee rate bucket, normalized to ten minutes. All zero when the + * snapshots span no measurable time, since inflow is only observable between two + * snapshots taken at the same block height. */ fun calculateInflows( mempoolSnapshots: List, @@ -72,9 +76,16 @@ internal object InflowCalculator { inflows += delta } - // Normalize inflows to 10 minutes - val tenMinutes = Duration.ofMinutes(10) - val normalizationFactor = tenMinutes.seconds.toDouble() / totalTimeSpan.seconds + // Normalize inflows to 10 minutes, i.e. one block's worth of arrivals. + // + // totalTimeSpan is zero whenever no block height carries more than one snapshot -- the ordinary + // state for a collector polling once per block. Every delta above is then zero too, so there is + // nothing to normalize. Dividing anyway yielded Infinity, then 0.0 * Infinity = NaN in every + // bucket, and those NaNs voided the whole fee table with no indication of why. + if (totalTimeSpan.isZero || totalTimeSpan.isNegative) return F64Array(bucketLayout.arraySize) + + // toMillis rather than seconds, which truncated sub-second spans to zero and divided by zero. + val normalizationFactor = TEN_MINUTES.toMillis().toDouble() / totalTimeSpan.toMillis() inflows *= normalizationFactor return inflows diff --git a/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorContractTest.kt b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorContractTest.kt new file mode 100644 index 0000000..ce45658 --- /dev/null +++ b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorContractTest.kt @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2025 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.block.augur + +import org.junit.jupiter.api.Test +import xyz.block.augur.test.TestUtils +import java.time.Instant +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Contract tests for [FeeEstimator]: what it rejects, and what it reports when it cannot answer. + * + * Each case here covers a way the estimator previously returned a plausible but wrong number + * instead of declining to answer. + */ +class FeeEstimatorContractTest { + private val snapshots = TestUtils.createSnapshotSequence(blockCount = 5, snapshotsPerBlock = 3) + + @Test + fun `an unanswerable confidence level reports null rather than the minimum fee rate`() { + // 1 - exp(-1) = 0.632 is the chance that any block at all is found in one block's time, so no + // fee rate achieves 90% confidence at a 1 block target. This used to report the lowest bucket, + // 1 sat/vB, which reads as a real and extremely cheap recommendation. + val estimator = FeeEstimator(probabilities = listOf(0.9), blockTargets = listOf(1.0)) + + assertNull(estimator.calculateEstimates(snapshots).getFeeRate(1, 0.9)) + } + + @Test + fun `an unanswerable short target does not clamp the longer targets in its column`() { + // Monotonicity walks targets in ascending order and clamps each rate to the previous one. When + // the 1 block cell collapsed to 1 sat/vB, every longer target inherited that bound and the + // whole column read 1 sat/vB. + val estimator = + FeeEstimator(probabilities = listOf(0.9), blockTargets = listOf(1.0, 6.0, 24.0, 144.0)) + val estimate = estimator.calculateEstimates(snapshots) + + assertNull(estimate.getFeeRate(1, 0.9)) + listOf(6, 24, 144).forEach { target -> + val feeRate = assertNotNull(estimate.getFeeRate(target, 0.9), "target=$target should be answerable") + assertTrue(feeRate > 1.0, "target=$target collapsed to the minimum fee rate: $feeRate") + } + } + + @Test + fun `raising maxFeeRate does not turn an unanswerable cell into a huge fee rate`() { + // Unanswerable cells used to be flagged with an out-of-range bucket index whose fee rate, + // 22247.84 sat/vB, only failed the output filter because it happened to sit above the default + // maxFeeRate. Raising the filter exposed it as a real estimate. + val estimator = + FeeEstimator( + probabilities = listOf(0.9), + blockTargets = listOf(1.0), + maxFeeRate = 1_000_000.0, + ) + + assertNull(estimator.calculateEstimates(snapshots).getFeeRate(1, 0.9)) + } + + @Test + fun `unsorted block targets give the same estimates as sorted ones`() { + // Monotonicity assumes ascending targets. With an unsorted list the longest target was visited + // first and became the bound for every shorter one. + val ascending = FeeEstimator(blockTargets = listOf(3.0, 12.0, 144.0)) + val shuffled = FeeEstimator(blockTargets = listOf(144.0, 3.0, 12.0)) + + assertEquals( + ascending.calculateEstimates(snapshots).estimates, + shuffled.calculateEstimates(snapshots).estimates, + ) + } + + @Test + fun `estimates stay monotonic past the long term window`() { + // The short/long blend ramp used to be a parabola that turned back down after the window: the + // long-term weight hit 0 again at 288 blocks and reached -35 at 1008, so the two estimates were + // extrapolated apart instead of averaged and longer targets could cost more than shorter ones. + val targets = listOf(144.0, 288.0, 576.0, 1008.0) + val estimate = + FeeEstimator(probabilities = listOf(0.5), blockTargets = targets) + .calculateEstimates(TestUtils.createSnapshotSequence(blockCount = 144, snapshotsPerBlock = 3)) + + var previous = Double.MAX_VALUE + targets.forEach { target -> + val feeRate = assertNotNull(estimate.getFeeRate(target.toInt(), 0.5), "target=$target") + assertTrue(feeRate <= previous, "target=$target cost $feeRate, more than the shorter target's $previous") + previous = feeRate + } + } + + @Test + fun `snapshots at distinct block heights report no estimates instead of a table of nulls`() { + // Inflow is the difference between two snapshots at the same height. One snapshot per height is + // the normal state for a per-block collector, and it left the inflow window spanning zero time: + // the division produced Infinity, then NaN, and every cell in the table came back null with no + // indication of why. + val start = Instant.now() + val oncePerBlock = + (0 until 6).map { i -> + TestUtils.createSnapshot( + blockHeight = 100 + i, + timestamp = start.plusSeconds(600L * i), + transactions = listOf(TestUtils.createTransaction(feeRate = 50.0, weight = 4_000_000)), + ) + } + + val estimate = FeeEstimator().calculateEstimates(oncePerBlock) + + assertTrue(estimate.estimates.isEmpty(), "expected no estimates, got ${estimate.estimates}") + assertEquals(oncePerBlock.last().timestamp, estimate.timestamp) + } + + @Test + fun `getNearestBlockTarget breaks ties toward the smaller target`() { + // Previously the winner of an exact tie was whichever key the map yielded first. + val estimate = + FeeEstimate( + estimates = mapOf( + 6 to BlockTarget(6, mapOf(0.5 to 20.0)), + 10 to BlockTarget(10, mapOf(0.5 to 10.0)), + ), + timestamp = Instant.now(), + ) + + assertEquals(6, estimate.getNearestBlockTarget(8)) + assertEquals(6, estimate.getNearestBlockTarget(7)) + assertEquals(10, estimate.getNearestBlockTarget(9)) + // Extreme inputs must not overflow the distance comparison. + assertEquals(6, estimate.getNearestBlockTarget(Int.MIN_VALUE)) + assertEquals(10, estimate.getNearestBlockTarget(Int.MAX_VALUE)) + } +} diff --git a/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt index 9fc3974..11055f1 100644 --- a/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt +++ b/lib/src/test/kotlin/xyz/block/augur/FeeEstimatorTest.kt @@ -20,6 +20,8 @@ import org.junit.jupiter.api.Test import xyz.block.augur.test.TestUtils import java.time.Duration import java.time.Instant +import kotlin.math.exp +import kotlin.random.Random import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertNull @@ -93,9 +95,10 @@ class FeeEstimatorTest { FeeEstimator.DEFAULT_PROBABILITIES.forEach { probability -> val feeRate = estimate.getFeeRate(target.toInt(), probability) if (feeRate != null) { - assert(feeRate >= lastFeeRate) { - "Fee rates should increase with probability for target=$target" - } + assertTrue( + feeRate >= lastFeeRate, + "Fee rates should increase with probability for target=$target", + ) lastFeeRate = feeRate } } @@ -118,9 +121,10 @@ class FeeEstimatorTest { FeeEstimator.DEFAULT_BLOCK_TARGETS.forEach { target -> val feeRate = estimate.getFeeRate(target.toInt(), probability) if (feeRate != null) { - assert(feeRate <= lastFeeRate) { - "Fee rates should decrease with target blocks for probability=$probability" - } + assertTrue( + feeRate <= lastFeeRate, + "Fee rates should decrease with target blocks for probability=$probability", + ) lastFeeRate = feeRate } } @@ -150,9 +154,10 @@ class FeeEstimatorTest { FeeEstimator.DEFAULT_BLOCK_TARGETS.forEach { target -> val feeRate = estimate.getFeeRate(target.toInt(), probability) if (feeRate != null) { - assert(feeRate <= lastFeeRate) { - "Fee rates should decrease with target blocks for probability=$probability" - } + assertTrue( + feeRate <= lastFeeRate, + "Fee rates should decrease with target blocks for probability=$probability", + ) lastFeeRate = feeRate } } @@ -177,12 +182,24 @@ class FeeEstimatorTest { val estimate = customEstimator.calculateEstimates(snapshots) - // Verify that estimates exist only for custom probabilities and targets + // Verify that estimates exist only for custom probabilities and targets. + // + // A cell is answerable only up to the chance that at least one block is found within the target, + // 1 - exp(-target). For a 1 block target that ceiling is 0.632, so 90% confidence in 1 block is + // unanswerable at any fee rate and must report null. This test used to assert a fee rate existed + // there, and it passed because the calculator substituted the lowest bucket -- so "confirm in + // one block, 90% sure" answered 1 sat/vB. customTargets.forEach { target -> customProbabilities.forEach { probability -> val feeRate = estimate.getFeeRate(target.toInt(), probability) - assert(feeRate != null && feeRate > 0.0) { - "Fee rate should exist for custom target=$target, probability=$probability" + if (probability > 1 - exp(-target)) { + assertNull(feeRate, "No fee rate can give $probability confidence within $target block(s)") + } else { + assertNotNull(feeRate, "Fee rate should exist for custom target=$target, probability=$probability") + assertTrue( + feeRate > 0.0, + "Fee rate should be positive for custom target=$target, probability=$probability", + ) } } } @@ -197,7 +214,7 @@ class FeeEstimatorTest { startTime = startTime, blockCount = 5, snapshotsPerBlock = 3, - ).shuffled() // Randomize order + ).shuffled(Random(TestUtils.FIXTURE_SEED)) // Fixed seed so a failure is reproducible val estimate = feeEstimator.calculateEstimates(snapshots) diff --git a/lib/src/test/kotlin/xyz/block/augur/internal/FeeEstimatesCalculatorTest.kt b/lib/src/test/kotlin/xyz/block/augur/internal/FeeEstimatesCalculatorTest.kt index c8f774c..65b8ddd 100644 --- a/lib/src/test/kotlin/xyz/block/augur/internal/FeeEstimatesCalculatorTest.kt +++ b/lib/src/test/kotlin/xyz/block/augur/internal/FeeEstimatesCalculatorTest.kt @@ -61,9 +61,11 @@ class FeeEstimatesCalculatorTest { } @Test - fun `test findBestIndex when no weights are fully mined`() { + fun `test findBestIndex returns null when no weights are fully mined`() { val weights = F64Array(5) { 1000.0 } - assertEquals(defaultLayout.bucketMax + 1, calculator.findBestIndex(weights)) + // Null rather than bucketMax + 1: an out-of-range bucket index is only distinguishable from a + // real one by remembering to range-check it, and one caller did not. + assertNull(calculator.findBestIndex(weights)) } @Test @@ -255,7 +257,7 @@ class FeeEstimatesCalculatorTest { } @Test - fun `test runSimulation ignores estimate when no buckets fully mined`() { + fun `test runSimulation returns null when no buckets fully mined`() { val initialWeights = F64Array(5) { 4.0 } val addedWeights = F64Array(5) { 4.0 } @@ -268,7 +270,7 @@ class FeeEstimatesCalculatorTest { blockSize = 1.0, ) - assertEquals(defaultLayout.bucketMax + 1, result) // Index > defaultLayout.bucketMax, indicating no estimate + assertNull(result, "no bucket was fully mined, so there is no estimate to report") } @Test @@ -391,4 +393,87 @@ class FeeEstimatesCalculatorTest { } } } + + @Test + fun `test mineBlock does not gain capacity from a negative bucket weight`() { + val weights = F64Array(4) { 1000.0 } + weights[1] = -10_000.0 + + val remaining = calculator.mineBlock(weights, blockSize = 1500.0) + + // Capacity is 1500: bucket 0 takes 1000, bucket 1 has nothing minable, bucket 2 takes the + // last 500 and bucket 3 is untouched. Treating -10000 as 10000 units of freed capacity used to + // let one block clear the rest of the ladder, so the whole mempool looked clearable at the + // minimum fee rate. + assertEquals(0.0, remaining[0]) + assertEquals(-10_000.0, remaining[1]) + assertEquals(500.0, remaining[2]) + assertEquals(1000.0, remaining[3]) + } + + @Test + fun `test enforceMonotonicity leaves unavailable cells out of the running bound`() { + // NaN marks a cell no fee rate can satisfy. It must not become the bound for the rest of the + // column: comparisons against NaN are always false, so the bound silently stuck at NaN and + // every longer target was overwritten with it. + val feeRates = F64Array(3, 1) + feeRates[0, 0] = Double.NaN + feeRates[1, 0] = 50.0 + feeRates[2, 0] = 40.0 + + val result = calculator.enforceMonotonicity(feeRates) + + assertTrue(result[0, 0].isNaN(), "the unanswerable cell should stay unanswerable") + assertEquals(50.0, result[1, 0]) + assertEquals(40.0, result[2, 0]) + } + + @Test + fun `test getWeightedEstimates saturates the long term weight past the window`() { + // The ramp 1 - (1 - t/window)^2 peaks at the window and then falls away: without saturation the + // long-term weight returns to 0 at twice the window and goes negative beyond, so the two + // estimates were extrapolated apart rather than blended. + val targets = listOf(144.0, 288.0, 1008.0) + val calc = FeeEstimatesCalculator(listOf(0.5), targets, BucketLayout.DEFAULT) + + val short = F64Array(targets.size, 1) + val long = F64Array(targets.size, 1) + for (i in targets.indices) { + short[i, 0] = 100.0 + long[i, 0] = 900.0 + } + + val weighted = calc.getWeightedEstimates(short, long) + + // At and past the window every target is the pure long-term estimate. + targets.indices.forEach { i -> + assertEquals(900.0, weighted[i, 0], 1e-9, "target=${targets[i]} should be the long-term estimate") + } + } + + @Test + fun `test getWeightedEstimates ramp reaches the long term estimate at the configured window`() { + // The window used to be hardcoded to 144 blocks no matter how FeeEstimator was configured, so a + // caller with a 12 hour long-term window still had the ramp stretched over 24 hours' worth. + val targets = listOf(72.0, 144.0) + val short = F64Array(targets.size, 1) + val long = F64Array(targets.size, 1) + for (i in targets.indices) { + short[i, 0] = 100.0 + long[i, 0] = 900.0 + } + + // 12 hour window: the ramp is complete at 72 blocks. + val twelveHour = + FeeEstimatesCalculator(listOf(0.5), targets, BucketLayout.DEFAULT, longTermWindowBlocks = 72.0) + .getWeightedEstimates(short, long) + assertEquals(900.0, twelveHour[0, 0], 1e-9) + + // 24 hour window: 72 blocks is only halfway, so the blend is 3/4 long-term. + val twentyFourHour = + FeeEstimatesCalculator(listOf(0.5), targets, BucketLayout.DEFAULT, longTermWindowBlocks = 144.0) + .getWeightedEstimates(short, long) + assertEquals(700.0, twentyFourHour[0, 0], 1e-9) + assertEquals(900.0, twentyFourHour[1, 0], 1e-9) + } } diff --git a/lib/src/test/kotlin/xyz/block/augur/internal/InflowCalculatorTest.kt b/lib/src/test/kotlin/xyz/block/augur/internal/InflowCalculatorTest.kt index 84483dd..f4e35e2 100644 --- a/lib/src/test/kotlin/xyz/block/augur/internal/InflowCalculatorTest.kt +++ b/lib/src/test/kotlin/xyz/block/augur/internal/InflowCalculatorTest.kt @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test import java.time.Duration import java.time.Instant import kotlin.test.assertEquals +import kotlin.test.assertTrue class InflowCalculatorTest { @Test @@ -184,4 +185,53 @@ class InflowCalculatorTest { assertEquals(BucketLayout.DEFAULT.arraySize, inflows.length) assertEquals(3000.0, inflows[0]) } + + @Test + fun `test calculateInflows returns zeros when every snapshot is at a distinct block height`() { + val now = Instant.now() + + // One snapshot per height is the normal state for a per-block collector. Inflow is only + // observable between two snapshots at the same height, so the measured span is zero. Dividing by + // it produced Infinity, then 0.0 * Infinity = NaN in every bucket, and those NaNs propagated + // through the simulation until every cell of the fee table came back null. + val snapshots = + (0 until 3).map { i -> + MempoolSnapshotF64Array( + now.plusSeconds(600L * i), + 100 + i, + F64Array(BucketLayout.DEFAULT.arraySize) { 1000.0 * (i + 1) }, + ) + } + + val inflows = + InflowCalculator.calculateInflows( + mempoolSnapshots = snapshots, + timeframe = Duration.ofHours(24), + ) + + assertEquals(BucketLayout.DEFAULT.arraySize, inflows.length) + assertEquals(0.0, inflows.sum()) + assertTrue(inflows.toDoubleArray().all { it.isFinite() }, "inflows must not contain NaN or Infinity") + } + + @Test + fun `test calculateInflows normalizes sub-second spans`() { + val now = Instant.now() + + // The span used to be measured in whole seconds, so snapshots less than a second apart truncated + // to a zero span and divided by zero. 500ms of +1000 is 1.2M per 10 minutes. + val snapshots = + listOf( + MempoolSnapshotF64Array(now, 100, F64Array(BucketLayout.DEFAULT.arraySize) { 1000.0 }), + MempoolSnapshotF64Array(now.plusMillis(500), 100, F64Array(BucketLayout.DEFAULT.arraySize) { 2000.0 }), + ) + + val inflows = + InflowCalculator.calculateInflows( + mempoolSnapshots = snapshots, + timeframe = Duration.ofMinutes(10), + ) + + assertEquals(1_200_000.0, inflows[0]) + } } diff --git a/lib/src/test/kotlin/xyz/block/augur/test/TestUtils.kt b/lib/src/test/kotlin/xyz/block/augur/test/TestUtils.kt index f62a27e..ed45ec6 100644 --- a/lib/src/test/kotlin/xyz/block/augur/test/TestUtils.kt +++ b/lib/src/test/kotlin/xyz/block/augur/test/TestUtils.kt @@ -23,6 +23,13 @@ import java.time.Instant import kotlin.random.Random object TestUtils { + /** + * Fixture weights are pseudo-random so the mempool has a realistic shape, but the seed is fixed: + * an unseeded generator gave every run a different mempool, so a failure could not be reproduced + * and assertions had to stay loose enough to pass on any draw. + */ + const val FIXTURE_SEED: Long = 20250101L + fun createSnapshot( blockHeight: Int, timestamp: Instant = Instant.now(), @@ -42,19 +49,22 @@ object TestUtils { return MempoolTransaction(weight = weight, fee = fee) } - // Default test data generators - fun createDefaultBaseWeights(): Map = - buildMap { + // Default test data generators. + // A fresh generator per call, not one shared instance, so the fixture does not depend on how many + // times it has already been built. + fun createDefaultBaseWeights(): Map { + val random = Random(FIXTURE_SEED) + return buildMap { // Low fee range (0.5 - 4.0 sat/vB) for (fee in 1..8) { val feeRate = fee * 0.5 - put(feeRate, (500_000L + (Random.nextDouble() * 1_500_000L).toLong())) + put(feeRate, (500_000L + (random.nextDouble() * 1_500_000L).toLong())) } // Medium fee range (4.5 - 16.0 sat/vB) for (fee in 9..32) { val feeRate = fee * 0.5 - val baseWeight = 2_000_000L + (Random.nextDouble() * 5_000_000L).toLong() + val baseWeight = 2_000_000L + (random.nextDouble() * 5_000_000L).toLong() val weight = when (feeRate) { 5.0 -> baseWeight * 3 // Spike at 5 sat/vB @@ -70,7 +80,7 @@ object TestUtils { // High fee range (16.5 - 32.0 sat/vB) for (fee in 33..64) { val feeRate = fee * 0.5 - val baseWeight = 1_000_000L + (Random.nextDouble() * 3_000_000L).toLong() + val baseWeight = 1_000_000L + (random.nextDouble() * 3_000_000L).toLong() val weight = when (feeRate) { 20.0 -> baseWeight * 3 // Spike at 20 sat/vB @@ -81,6 +91,7 @@ object TestUtils { put(feeRate, weight) } } + } fun createHighInflowRates(): Map = buildMap {