Report null instead of a wrong fee rate when an estimate is unavailable

lib/api/lib.api

@@ -32,6 +32,7 @@ public final class xyz/block/augur/FeeEstimate {
 
 public final class xyz/block/augur/FeeEstimator {
 	public static final field Companion Lxyz/block/augur/FeeEstimator$Companion;
+	public static final field MAX_BLOCK_TARGET D
 	public fun <init> ()V
 	public fun <init> (Ljava/util/List;)V
 	public fun <init> (Ljava/util/List;Ljava/util/List;)V

lib/src/main/kotlin/xyz/block/augur/FeeEstimate.kt

@@ -69,15 +69,21 @@ public data class FeeEstimate(
    *
    * This is useful when the exact requested block target is not available.
    *
+   * Ties are broken toward the smaller target, which carries a fee rate at least as high.
+   *
    * @param targetBlocks The desired confirmation target in blocks
    * @return The nearest available block target, or null if no estimates are available
    */
   public fun getNearestBlockTarget(targetBlocks: Int): Int? {
     if (estimates.isEmpty()) return null
     if (estimates.containsKey(targetBlocks)) return targetBlocks
 
-    return estimates.keys
-      .minByOrNull { kotlin.math.abs(it - targetBlocks) }
+    // Comparing in Long avoids overflow on extreme inputs, and the second comparator makes the
+    // result independent of map iteration order -- previously an exact tie returned whichever key
+    // the map happened to yield first.
+    return estimates.keys.minWithOrNull(
+      compareBy({ kotlin.math.abs(it.toLong() - targetBlocks) }, { it }),
+    )
   }
 
   /**

lib/src/main/kotlin/xyz/block/augur/FeeEstimator.kt

@@ -41,8 +41,11 @@ import java.time.Instant
  * val feeRate = estimate.getFeeRate(targetBlocks = 6, probability = 0.95)
  * ```
  *
- * @property probabilities The confidence levels to calculate (default: 5%, 20%, 50%, 80%, 95%)
- * @property blockTargets The block confirmation targets to estimate for (default: 3, 6, 9, 12, 18, 24, 36, 48, 72, 96, 144)
+ * @property probabilities The confidence levels to calculate (default: 5%, 20%, 50%, 80%, 95%).
+ *   A level above `1 - exp(-blockTarget)` — the chance any block is found within the target, 0.9502
+ *   at 3 blocks — is unanswerable at any fee rate and estimates null for that cell.
+ * @property blockTargets The block confirmation targets to estimate for (default: 3, 6, 9, 12, 18,
+ *   24, 36, 48, 72, 96, 144). Sorted internally.
  * @property minFeeRate The minimum fee rate in sat/vB for the simulation lower bound (default: 1.0).
  *   Set to 0.1 for Bitcoin Core 29.1/30.0+ nodes that support sub-1 sat/vB fee rates. Snapshots
  *   store all bucketed transactions regardless of this value; `minFeeRate` controls which buckets
@@ -58,25 +61,42 @@ import java.time.Instant
  */
 public class FeeEstimator @JvmOverloads public constructor(
   private val probabilities: List<Double> = DEFAULT_PROBABILITIES,
-  private val blockTargets: List<Double> = DEFAULT_BLOCK_TARGETS,
+  blockTargets: List<Double> = DEFAULT_BLOCK_TARGETS,
   private val shortTermWindowDuration: Duration = Duration.ofMinutes(30),
   private val longTermWindowDuration: Duration = Duration.ofHours(24),
   private val minFeeRate: Double = DEFAULT_MIN_FEE_RATE,
   private val maxFeeRate: Double = DEFAULT_MAX_FEE_RATE,
 ) {
+  /**
+   * Sorted ascending. [FeeEstimatesCalculator.enforceMonotonicity] walks block targets in order, so
+   * an unsorted list used to clamp every short target down to the longest target's fee rate.
+   */
+  private val blockTargets: List<Double> = blockTargets.sorted()
+
+  /** The long-term window in blocks, where the short/long blend ramp reaches a pure long-term estimate. */
+  private val longTermWindowBlocks: Double =
+    longTermWindowDuration.toMillis() / (FeeEstimatesCalculator.MINUTES_PER_BLOCK * 60_000.0)
+
   private val bucketLayout: BucketLayout
   private val feeEstimatesCalculator: FeeEstimatesCalculator
 
   init {
     require(probabilities.isNotEmpty()) { "At least one probability level must be provided" }
-    require(blockTargets.isNotEmpty()) { "At least one block target must be provided" }
+    require(this.blockTargets.isNotEmpty()) { "At least one block target must be provided" }
     require(probabilities.all { it in 0.0..1.0 }) { "All probabilities must be between 0.0 and 1.0" }
-    require(blockTargets.all { it > 0 }) { "All block targets must be positive" }
+    require(this.blockTargets.all { it > 0 }) { "All block targets must be positive" }
+    // Same cost bound as numOfBlocks below: a target of a million blocks simulates a million blocks.
+    require(this.blockTargets.all { it <= MAX_BLOCK_TARGET }) {
+      "All block targets must be at most $MAX_BLOCK_TARGET, was ${this.blockTargets.filter { it > MAX_BLOCK_TARGET }}"
+    }
     require(maxFeeRate > 0.0) { "maxFeeRate must be positive, was $maxFeeRate" }
     bucketLayout = BucketLayout(minFeeRate)
-    feeEstimatesCalculator = FeeEstimatesCalculator(probabilities, blockTargets, bucketLayout, maxFeeRate)
+    feeEstimatesCalculator = newCalculator(this.blockTargets)
   }
 
+  private fun newCalculator(targets: List<Double>): FeeEstimatesCalculator =
+    FeeEstimatesCalculator(probabilities, targets, bucketLayout, maxFeeRate, longTermWindowBlocks)
+
   /**
    * Calculates fee estimates based on historical mempool snapshots.
    *
@@ -93,6 +113,12 @@ public class FeeEstimator @JvmOverloads public constructor(
     // If numOfBlocks is specified then it needs to be at least 3,
     // since we can't simulate partial blocks being mined
     require(numOfBlocks == null || numOfBlocks >= 3.0) { "numOfBlocks must be at least 3 if specified" }
+    // It also needs an upper bound: getExpectedBlocksMined evaluates a Poisson tail of
+    // 4 * numOfBlocks entries and then simulates that many blocks, so an unbounded value costs
+    // minutes of CPU per call -- and callers behind an HTTP service pass a request parameter through.
+    require(numOfBlocks == null || numOfBlocks <= MAX_BLOCK_TARGET) {
+      "numOfBlocks must be at most $MAX_BLOCK_TARGET if specified, was $numOfBlocks"
+    }
 
     if (mempoolSnapshots.isEmpty()) {
       return FeeEstimate(emptyMap(), Instant.now())
@@ -102,13 +128,20 @@ public class FeeEstimator @JvmOverloads public constructor(
     val orderedSnapshots = mempoolSnapshots.sortedBy { it.timestamp }
     val simdSnapshots = orderedSnapshots.map { MempoolSnapshotF64Array.fromMempoolSnapshot(it, bucketLayout) }
 
+    // Inflow is only observable between two snapshots at the same block height, so without such a
+    // pair there is no estimate to make. Previously this divided by a zero span and returned a table
+    // of nulls instead of saying so.
+    if (simdSnapshots.groupingBy { it.blockHeight }.eachCount().none { it.value > 1 }) {
+      return FeeEstimate(emptyMap(), orderedSnapshots.last().timestamp)
+    }
+
     // Extract latest mempool weights and calculate inflow rates
     val latestMempoolWeights = simdSnapshots.last().buckets
     val shortTermInflows = InflowCalculator.calculateInflows(simdSnapshots, shortTermWindowDuration, bucketLayout)
     val longTermInflows = InflowCalculator.calculateInflows(simdSnapshots, longTermWindowDuration, bucketLayout)
 
     val (calculator, targets) = if (numOfBlocks != null) {
-      FeeEstimatesCalculator(probabilities, listOf(numOfBlocks), bucketLayout, maxFeeRate) to listOf(numOfBlocks)
+      newCalculator(listOf(numOfBlocks)) to listOf(numOfBlocks)
     } else {
       feeEstimatesCalculator to blockTargets
     }
@@ -199,5 +232,11 @@ public class FeeEstimator @JvmOverloads public constructor(
      * simulation ceiling (bucket 1000) pass the filter.
      */
     public val DEFAULT_MAX_FEE_RATE: Double = FeeEstimatesCalculator.DEFAULT_MAX_FEE_RATE
+
+    /**
+     * Largest supported block target, one week of blocks. Simulation cost grows linearly with the
+     * target, so this bound keeps a caller-supplied `numOfBlocks` from costing minutes of CPU.
+     */
+    public const val MAX_BLOCK_TARGET: Double = 1008.0
   }
 }

lib/src/main/kotlin/xyz/block/augur/internal/FeeEstimatesCalculator.kt

@@ -33,6 +33,7 @@ internal class FeeEstimatesCalculator(
   private val blockTargets: List<Double>,
   private val bucketLayout: BucketLayout = BucketLayout.DEFAULT,
   private val maxFeeRate: Double = DEFAULT_MAX_FEE_RATE,
+  private val longTermWindowBlocks: Double = DEFAULT_LONG_TERM_WINDOW_BLOCKS,
 ) {
   private val expectedBlocksMined by lazy { getExpectedBlocksMined() }
 
@@ -43,7 +44,8 @@ internal class FeeEstimatesCalculator(
    * @param shortIntervalInflows Short-term inflow data (typically 30 minutes)
    * @param longIntervalInflows Long-term inflow data (typically 24 hours)
    * @return A 2D array of fee estimates where each element corresponds to a specific
-   *         block target and probability level. Values exceeding [maxFeeRate] are null.
+   *         block target and probability level. An element is null when it exceeds [maxFeeRate],
+   *         or when no fee rate can satisfy that (block target, probability) pair at all.
    */
   fun getFeeEstimates(
     mempoolSnapshot: F64Array,
@@ -96,13 +98,16 @@ internal class FeeEstimatesCalculator(
       probabilities.indices.forEach { probIndex ->
         val expectedBlocks = expectedBlocksMined[blockTargetIndex, probIndex].toInt()
 
-        // Run individual simulation and store result
+        // Run individual simulation and store result. NaN marks "no fee rate satisfies this pair"
+        // and prepareResultArray turns it into null. Bucket 0 must never stand in for failure: it is
+        // a real fee rate (minFeeRate, 1 sat/vB by default), so a caller could not tell a genuine
+        // 1 sat/vB recommendation apart from a failed simulation.
         result[blockTargetIndex, probIndex] = runSimulation(
           initialWeights,
           addedWeights,
           expectedBlocks,
           meanBlocks,
-        )?.toDouble() ?: 0.0
+        )?.toDouble() ?: Double.NaN
       }
     }
 
@@ -111,7 +116,9 @@ internal class FeeEstimatesCalculator(
 
   /**
    * Simulates mining blocks and returns the weight index corresponding to the
-   * lowest fee rate that would result in the transaction getting mined.
+   * lowest fee rate that would result in the transaction getting mined, or null when no
+   * simulation is possible (no blocks are expected to be mined at this confidence level) or
+   * when even the highest fee rate bucket would not clear.
    */
   internal fun runSimulation(
     initialWeights: F64Array,
@@ -154,17 +161,20 @@ internal class FeeEstimatesCalculator(
     var weightUnitsRemaining = blockSize
 
     for (i in 0 until weightsRemaining.length) {
-      val removedWeight = min(weightsRemaining[i], weightUnitsRemaining)
+      // coerceAtLeast(0.0) stops a negative bucket weight from *adding* to the block's remaining
+      // capacity, which would otherwise let a single block mine more than blockSize weight units
+      // and make the whole mempool look clearable at the minimum fee rate.
+      val removedWeight = min(weightsRemaining[i], weightUnitsRemaining).coerceAtLeast(0.0)
       weightUnitsRemaining -= removedWeight
       weightsRemaining[i] -= removedWeight
     }
     return weightsRemaining
   }
 
   /**
-   * Find the index of the last bucket that is fully mined.
+   * Find the index of the last bucket that is fully mined, or null if no bucket is.
    */
-  internal fun findBestIndex(weightsRemaining: F64Array): Int {
+  internal fun findBestIndex(weightsRemaining: F64Array): Int? {
     // The last mined bucket will occur just before the first non-zero remaining weight.
     val index = weightsRemaining.toDoubleArray().indexOfFirst { it != 0.0 } - 1
 
@@ -173,7 +183,7 @@ internal class FeeEstimatesCalculator(
     // Else, createFeeRateBuckets reversed the order, so subtract to recover the original index.
     return when (index) {
       -2 -> bucketLayout.bucketMin // all weights are zero so we can use the cheapest fee rate
-      -1 -> bucketLayout.bucketMax + 1 // return null
+      -1 -> null // not even the highest fee rate bucket cleared, so no answer exists
       else -> bucketLayout.toBucketIndex(index)
     }
   }
@@ -185,9 +195,13 @@ internal class FeeEstimatesCalculator(
     shortEstimates: F64Array,
     longEstimates: F64Array,
   ): F64Array {
-    // The longer estimates are weighted more heavily for longer intervals. For example, the
-    // weighted estimate for 24 hours (144 blocks) is exactly equal to the longEstimate.
-    val weights = blockTargets.map { 1 - (1 - it / 144.0).pow(2) }
+    // The longer estimates are weighted more heavily for longer intervals, reaching a pure
+    // long-term estimate at the long-term window (144 blocks for the default 24 hours).
+    //
+    // coerceAtMost(1.0) saturates the ramp past that point. Without it the parabola turns back down,
+    // returning to 0 at twice the window and reaching -35 at 1008 blocks, which extrapolates the two
+    // estimates apart instead of averaging them.
+    val weights = blockTargets.map { 1 - (1 - (it / longTermWindowBlocks).coerceAtMost(1.0)).pow(2) }
     val weightedEstimates = F64Array(shortEstimates.shape[0], shortEstimates.shape[1])
 
     for (i in 0 until weightedEstimates.shape[0]) {
@@ -203,13 +217,18 @@ internal class FeeEstimatesCalculator(
   internal fun convertBucketsToFeeRates(bucketEstimates: F64Array): F64Array = (bucketEstimates / 100.0).exp()
 
   /**
-   * Converts fee estimates to the final nullable array format and filters fees above the maximum bucket's fee rate.
+   * Converts fee estimates to the final nullable array format, dropping cells that no fee rate can
+   * satisfy along with any fee above [maxFeeRate].
    * F64Array can't accommodate nulls so we convert to traditional arrays.
    */
   private fun prepareResultArray(feeRates: F64Array): Array<Array<Double?>> {
     return Array(feeRates.shape[0]) { blockTargetIndex ->
       Array(feeRates.shape[1]) { probabilityIndex ->
-        feeRates[blockTargetIndex, probabilityIndex].takeIf { it <= maxFeeRate }
+        // isFinite() drops the NaN markers runSimulations writes for unanswerable cells, and stays
+        // independent of maxFeeRate. The old out-of-band bucket sentinel did not: it relied on the
+        // gap between exp(1001/100) and DEFAULT_MAX_FEE_RATE, so raising maxFeeRate surfaced
+        // 22247.84 sat/vB as a real estimate.
+        feeRates[blockTargetIndex, probabilityIndex].takeIf { it.isFinite() && it <= maxFeeRate }
       }
     }
   }
@@ -241,12 +260,18 @@ internal class FeeEstimatesCalculator(
    * Ensures that fee rates decrease (or stay the same) as block targets increase.
    * For each probability, if a fee rate is higher than the previous one,
    * it is set equal to the previous rate.
+   *
+   * Assumes [blockTargets] is in ascending order; [xyz.block.augur.FeeEstimator] sorts it.
    */
   internal fun enforceMonotonicity(feeRates: F64Array): F64Array {
     val result = feeRates.copy()
     for (j in 0 until result.shape[1]) {
       var prevRate = Double.POSITIVE_INFINITY
       for (i in 0 until result.shape[0]) {
+        // Skip unavailable cells, and in particular don't let one become the running bound: a
+        // single unanswerable short target would otherwise clamp every longer target in the
+        // column down to it, discarding estimates that were computed correctly.
+        if (result[i, j].isNaN()) continue
         if (result[i, j] > prevRate) {
           result[i, j] = prevRate
         }
@@ -259,6 +284,12 @@ internal class FeeEstimatesCalculator(
   companion object {
     const val BLOCK_SIZE_WEIGHT_UNITS = 4_000_000
 
+    /** Bitcoin's target block interval, used to convert window durations into block counts. */
+    const val MINUTES_PER_BLOCK = 10.0
+
+    /** Blocks in the default 24 hour long-term window: 24 * 60 / 10. */
+    const val DEFAULT_LONG_TERM_WINDOW_BLOCKS = 144.0
+
     // Rounded up from exp(10) ≈ 22026.47 so estimates at the simulation ceiling pass the <= filter
     const val DEFAULT_MAX_FEE_RATE = 22027.0
   }

lib/src/main/kotlin/xyz/block/augur/internal/InflowCalculator.kt

@@ -26,12 +26,16 @@ import java.time.Duration
  * during the time period being estimated.
  */
 internal object InflowCalculator {
+  private val TEN_MINUTES: Duration = Duration.ofMinutes(10)
+
   /**
    * Calculates inflow rates based on historical snapshots.
    *
    * @param mempoolSnapshots List of mempool snapshots
    * @param timeframe Duration to consider for inflow calculation
-   * @return Array of inflow rates by fee rate bucket
+   * @return Array of inflow rates by fee rate bucket, normalized to ten minutes. All zero when the
+   *         snapshots span no measurable time, since inflow is only observable between two
+   *         snapshots taken at the same block height.
    */
   fun calculateInflows(
     mempoolSnapshots: List<MempoolSnapshotF64Array>,
@@ -72,9 +76,16 @@ internal object InflowCalculator {
       inflows += delta
     }
 
-    // Normalize inflows to 10 minutes
-    val tenMinutes = Duration.ofMinutes(10)
-    val normalizationFactor = tenMinutes.seconds.toDouble() / totalTimeSpan.seconds
+    // Normalize inflows to 10 minutes, i.e. one block's worth of arrivals.
+    //
+    // totalTimeSpan is zero whenever no block height carries more than one snapshot -- the ordinary
+    // state for a collector polling once per block. Every delta above is then zero too, so there is
+    // nothing to normalize. Dividing anyway yielded Infinity, then 0.0 * Infinity = NaN in every
+    // bucket, and those NaNs voided the whole fee table with no indication of why.
+    if (totalTimeSpan.isZero || totalTimeSpan.isNegative) return F64Array(bucketLayout.arraySize)
+
+    // toMillis rather than seconds, which truncated sub-second spans to zero and divided by zero.
+    val normalizationFactor = TEN_MINUTES.toMillis().toDouble() / totalTimeSpan.toMillis()
     inflows *= normalizationFactor
 
     return inflows