feat(dataset): admit coinbase overpayments at 197438 and 584802

README.md

@@ -55,9 +55,12 @@ It checks JSONL structure, types, ordering, uniqueness, header hash, PoW and dec
 It enforces the rule/reject-string mapping, required context and rule-specific predicates.
 Validation reports the first error in each record, with its file and line number, then continues to the next record.
 Available block files must parse completely and match their transaction merkle roots and applicable witness commitments.
-CI checks output-value overflow, forward transaction spends, excessive sigop cost and transaction reuse from the canonical parent directly.
+CI checks output-value overflow, forward transaction spends, excessive sigop cost, transaction reuse from the canonical parent and coinbase overpayment directly.
 
-Sigops, missing-parent and parent-transaction reuse checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
+For coinbase overpayment, CI compares the coinbase output sum with the subsidy at the record height plus fees calculated from authenticated previous output values.
+Coinbase-only bodies have zero fees and need no previous transactions.
+
+Sigops, missing-parent, parent-transaction reuse and fee accounting checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
 Missing entries are fetched from public Esplora-compatible APIs when `--fetch-prevouts` is supplied.
 API failures, missing evidence and corrupt cache entries fail validation.
 After filling the cache, omit the flag for an offline run; `--prevouts-dir` selects another cache and `--api-url` selects an API base.

ci/block_evidence.py

@@ -16,7 +16,7 @@
 from collections.abc import Mapping, Sequence
 from typing import TypeVar
 
-from bitcoin.core import CBlock, CoreMainParams, CTransaction, Hash as sha256d, b2lx, lx
+from bitcoin.core import COIN, CBlock, CoreMainParams, CTransaction, Hash as sha256d, MoneyRange, b2lx, lx
 from bitcoin.core.script import (
     CScript, CScriptInvalidError, CScriptOp, OP_1, OP_16,
     OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY,
@@ -179,6 +179,50 @@ def witness_sigops(program_script: bytes, witness: Sequence[bytes]) -> int:
     return 0
 
 
+def coinbase_amounts(block: CBlock, height: int,
+                     previous_transactions: Mapping[bytes, CTransaction]) -> dict[str, int]:
+    """Compare coinbase value with mainnet subsidy plus authenticated input fees.
+
+    The caller binds the height and supplies txid-verified external
+    transactions; earlier non-coinbase in-block outputs can also fund fees.
+    This does not prove historical unspentness, maturity or script validity.
+    Ambiguous spends and invalid amounts fail instead of lowering the allowance.
+    """
+    subsidy = (50 * COIN) >> (height // CoreMainParams.SUBSIDY_HALVING_INTERVAL)
+    txids = [tx.GetTxid() for tx in block.vtx]
+    available = dict(previous_transactions)
+    spent = set()
+    fees = 0
+    for index, tx in enumerate(block.vtx):
+        output_value = sum(output.nValue for output in tx.vout)
+        if any(not MoneyRange(output.nValue) for output in tx.vout) or not MoneyRange(output_value):
+            raise ValueError("output value outside money range")
+        if index == 0:
+            coinbase_value = output_value
+            continue
+        input_value = 0
+        for txin in tx.vin:
+            txid, vout = txin.prevout.hash, txin.prevout.n
+            if (txid, vout) in spent:
+                raise ValueError("repeated input in fee evidence")
+            previous = available.get(txid)
+            if previous is None or vout >= len(previous.vout):
+                raise ValueError(f"missing previous output {b2lx(txid)}:{vout}")
+            value = previous.vout[vout].nValue
+            if not MoneyRange(value):
+                raise ValueError("previous output value outside money range")
+            input_value += value
+            spent.add((txid, vout))
+        if not MoneyRange(input_value) or input_value < output_value:
+            raise ValueError("invalid transaction input value or negative fee")
+        fees += input_value - output_value
+        if not MoneyRange(fees):
+            raise ValueError("total fees outside money range")
+        available[txids[index]] = tx
+    return {"coinbase": coinbase_value, "subsidy": subsidy, "fees": fees,
+            "excess": coinbase_value - subsidy - fees}
+
+
 def sigop_cost(block: CBlock, previous_transactions: Mapping[bytes, CTransaction]) -> dict[str, int]:
     """Calculate legacy, P2SH and witness costs, requiring every prevout.
 

ci/sanity-check.py

@@ -22,7 +22,7 @@
 from bitcoin.core.serialize import uint256_from_compact
 
 from block_evidence import (
-    MAX_BLOCK_SIGOPS_COST, confirmed_at_or_after, establishes_rule, omitted_prevouts,
+    MAX_BLOCK_SIGOPS_COST, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts,
     read_block, reuses_parent_transaction, sigop_cost, verify_witness_commitment,
 )
 from prevouts import (
@@ -48,11 +48,14 @@
 # Evidence paths: local = header/context only; body = complete block file;
 # sigops = body plus previous transactions; missing_parent = body plus API
 # evidence for the recorded outpoint; parent_txid_reuse = body plus an
-# authenticated canonical parent txid list. Rule names and reject strings must
+# authenticated canonical parent txid list; cb_amount = body plus canonical
+# parent and fee prevouts.
+# Rule names and reject strings must
 # match docs/schema.md.
 RULES = {
     "bad-txns-vout-toolarge": ("bad-txns-vout-toolarge", (), "body"),
     "bad-blk-sigops": ("bad-blk-sigops", (), "sigops"),
+    "bad-cb-amount": ("bad-cb-amount", ("coinbase_scriptsig_hex", "parent_kind"), "cb_amount"),
     "bad-txns-inputs-missingorspent": ("bad-txns-inputs-missingorspent", (), "body"),
     "missing_unconfirmed_parent": ("bad-txns-inputs-missingorspent", ("missing_prevout",), "missing_parent"),
     "already_confirmed_in_parent": ("bad-txns-inputs-missingorspent",
@@ -317,6 +320,15 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
         parent_txids = load_parent_txids(record["prev_hash"], prevouts_dir, fetch_prevouts, apis)
         if not reuses_parent_transaction(block, parent_txids, record["context"]["parent_txid"]):
             raise ValueError("parent_txid is not a non-coinbase transaction in both the body and its parent")
+    if mode == "cb_amount":
+        require_canonical_parent(record, prevouts_dir, fetch_prevouts, apis)
+        previous = load_previous(block.vtx, prevouts_dir, fetch_prevouts, apis)
+        amounts = coinbase_amounts(block, record["height"], previous)
+        if amounts["excess"] <= 0:
+            raise ValueError("coinbase value does not exceed subsidy plus fees")
+        if fetch_prevouts:
+            print(f"{record['height']}: coinbase {amounts['coinbase']}, subsidy {amounts['subsidy']}, "
+                  f"fees {amounts['fees']}, excess {amounts['excess']} sat", flush=True)
     if mode == "missing_parent":
         txid, vout = record["context"]["missing_prevout"].split(":")
         if (lx(txid), int(vout)) not in omitted_prevouts(block.vtx):

ci/test_block_evidence.py

@@ -6,8 +6,8 @@
 from bitcoin.core.script import CScript, CScriptWitness, OP_TRUE
 
 from block_evidence import (
-    MAX_MONEY, confirmed_at_or_after, establishes_rule, omitted_prevouts, read_block, sha256d,
-    reuses_parent_transaction, sigop_count, witness_sigops,
+    MAX_MONEY, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts, read_block, read_transaction,
+    reuses_parent_transaction, sha256d, sigop_count, witness_sigops,
 )
 
 
@@ -102,6 +102,49 @@ def test_parent_transaction_reuse_excludes_coinbases_and_absent_transactions(sel
                 self.assertEqual(reuses_parent_transaction(body, parent, witness), expected)
 
 
+class CoinbaseAmountChecks(unittest.TestCase):
+    def test_subsidy_boundaries(self):
+        """Require a strict overpayment, including halving and zero-subsidy boundaries."""
+        for height, subsidy in ((209999, 5_000_000_000), (210000, 2_500_000_000),
+                                (584802, 1_250_000_000), (64 * 210000, 0)):
+            for excess in (0, 1):
+                with self.subTest(height=height, excess=excess):
+                    body = read_block(block(transaction(amount=subsidy + excess)))
+                    self.assertEqual(coinbase_amounts(body, height, {}),
+                                     {"coinbase": subsidy + excess, "subsidy": subsidy,
+                                      "fees": 0, "excess": excess})
+
+    def test_authenticated_fees_and_in_block_spends(self):
+        """Account for external and earlier in-block outputs without fetching internal ones."""
+        parent = read_transaction(transaction(prev_hash=b"\x11" * 32, vout=0, amount=100)[0])
+        first = transaction(prev_hash=parent.GetTxid(), vout=0, amount=90)
+        second = transaction(prev_hash=sha256d(first[1]), vout=0, amount=80)
+        for excess in (0, 1):
+            with self.subTest(excess=excess):
+                body = read_block(block(transaction(amount=1_250_000_020 + excess), first, second))
+                amounts = coinbase_amounts(body, 584802, {parent.GetTxid(): parent})
+                self.assertEqual(amounts["fees"], 20)
+                self.assertEqual(amounts["excess"], excess)
+
+    def test_unusable_fee_evidence_raises(self):
+        """Missing, repeated or negative-fee inputs cannot prove overpayment."""
+        parent = read_transaction(transaction(prev_hash=b"\x11" * 32, vout=0, amount=100)[0])
+        first = transaction(prev_hash=parent.GetTxid(), vout=0, amount=90)
+        second = transaction(prev_hash=sha256d(first[1]), vout=0, amount=80)
+        previous = {parent.GetTxid(): parent}
+        cb = transaction(amount=1_250_000_021)
+        cases = {
+            "missing": (block(cb, first), {}, "missing previous output"),
+            "missing vout": (block(cb, transaction(prev_hash=parent.GetTxid(), vout=1)), previous, "missing previous output"),
+            "double spend": (block(cb, first, transaction(prev_hash=parent.GetTxid(), vout=0, amount=80)), previous, "repeated input"),
+            "negative fee": (block(cb, transaction(prev_hash=parent.GetTxid(), vout=0, amount=101)), previous, "negative fee"),
+            "negative output": (block(transaction(amount=-1)), {}, "money range"),
+        }
+        for case, (raw, prevouts, error) in cases.items():
+            with self.subTest(case=case), self.assertRaisesRegex(ValueError, error):
+                coinbase_amounts(read_block(raw), 584802, prevouts)
+
+
 class SigopChecks(unittest.TestCase):
     def test_legacy_multisig_accurate_count_and_pushed_bytes(self):
         """Check multisig counting modes, ignore pushed opcodes and stop at malformed pushes."""

ci/test_sanity_check.py

@@ -114,6 +114,18 @@ def test_rule_context_required(self):
                     del self.record["context"][field]
                     self.assertTrue(any("requires" in p for p in self.validate()))
 
+    def test_coinbase_overpayment_evidence(self):
+        """Prove 584802 with no previous transactions, then both overpayments from the cached fee evidence."""
+        records = {r["height"]: copy.deepcopy(r) for r in self.records if r["rule"] == "bad-cb-amount"}
+        self.record = records[584802]
+        self.copy_body(self.record)
+        cache = self.root / "empty-cache"
+        with patch.object(CHECK, "load_canonical_hash", return_value=self.record["prev_hash"]), \
+                patch("prevouts.urlopen", side_effect=AssertionError("unnecessary download")):
+            self.assertEqual(self.validate(prevouts_dir=cache), [])
+        self.copy_body(records[197438])
+        self.assertEqual(self.validate([records[197438], self.record]), [])
+
     def test_sigops_requires_previous_transactions(self):
         """A complete sigops block still fails admission when previous transactions are missing."""
         self.record = self.for_rule("bad-blk-sigops")