feat(dataset): admit four 2018 AntPool parent-reuse blocks

.github/workflows/sanitycheck.yml

@@ -27,8 +27,10 @@ jobs:
         uses: actions/cache/restore@v6
         with:
           path: .cache/prevouts
-          key: prevouts-v1-${{ hashFiles('blocks/*.bin') }}
-          restore-keys: prevouts-v1-
+          key: prevouts-v2-${{ hashFiles('blocks/*.bin') }}
+          restore-keys: |
+            prevouts-v2-
+            prevouts-v1-
       - name: validate data and test validator
         run: |
           python -m venv .venv

README.md

@@ -34,6 +34,7 @@ Prefer immutable evidence URLs.
 For header rules, observations and full block files are optional.
 Body failures require a complete `.bin` that demonstrates the named failure.
 For sigops, CI fetches the referenced previous transactions from public APIs, verifies their transaction IDs, and calculates the cost using their output scripts.
+For `already_confirmed_in_parent`, CI checks that a named non-coinbase transaction also appears in the canonical parent, using a txid list authenticated against the parent's header merkle root.
 The [schema](docs/schema.md#evidence-enforced-by-ci) specifies each rule's evidence contract; observation labels cannot substitute for these checks.
 
 Replaying a `.bin` with `bitcoin-cli submitblock` reproduces context-free failures such as 74638's `bad-txns-vout-toolarge`; connect-level failures such as `bad-blk-sigops` need the historical chain context.
@@ -54,11 +55,11 @@ It checks JSONL structure, types, ordering, uniqueness, header hash, PoW and dec
 It enforces the rule/reject-string mapping, required context and rule-specific predicates.
 Validation reports the first error in each record, with its file and line number, then continues to the next record.
 Available block files must parse completely and match their transaction merkle roots and applicable witness commitments.
-CI checks output-value overflow, forward transaction spends and excessive sigop cost directly.
+CI checks output-value overflow, forward transaction spends, excessive sigop cost and transaction reuse from the canonical parent directly.
 
-Sigops and missing-parent checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
+Sigops, missing-parent and parent-transaction reuse checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
 Missing entries are fetched from public Esplora-compatible APIs when `--fetch-prevouts` is supplied.
-API failures, missing evidence and corrupt cached transactions fail validation.
+API failures, missing evidence and corrupt cache entries fail validation.
 After filling the cache, omit the flag for an offline run; `--prevouts-dir` selects another cache and `--api-url` selects an API base.
 The [schema](docs/schema.md#sigops-evidence-and-public-apis) explains the authentication and counting checks.
 

ci/block_evidence.py

@@ -16,7 +16,7 @@
 from collections.abc import Mapping, Sequence
 from typing import TypeVar
 
-from bitcoin.core import CBlock, CoreMainParams, CTransaction, Hash as sha256d, b2lx
+from bitcoin.core import CBlock, CoreMainParams, CTransaction, Hash as sha256d, b2lx, lx
 from bitcoin.core.script import (
     CScript, CScriptInvalidError, CScriptOp, OP_1, OP_16,
     OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY,
@@ -105,6 +105,15 @@ def confirmed_at_or_after(confirmation: tuple[int, str], height: int, block_hash
     return confirmed_height >= height and confirmed_hash != block_hash
 
 
+def reuses_parent_transaction(block: CBlock, parent_txids: Sequence[str], txid: str) -> bool:
+    """Require the named transaction in both blocks, excluding both coinbases.
+
+    The caller authenticates the ordered parent list and its canonical height.
+    """
+    return (txid in parent_txids[1:]
+            and any(not tx.is_coinbase() and tx.GetTxid() == lx(txid) for tx in block.vtx[1:]))
+
+
 def establishes_rule(block: CBlock, rule: str) -> bool:
     """Recognize only failures provable from these committed transactions.
 

ci/prevouts.py

@@ -1,11 +1,13 @@
-"""Fetch previous transactions, parent confirmations and canonical block hashes.
+"""Fetch transactions, confirmations, canonical hashes and authenticated parent txid lists.
 
 `{txid}.bin` is a stripped transaction, checked against its txid.
 `{txid}.status.json` is the Esplora status reply for that transaction, and
 `height-{n}.hash` is the block hash Esplora reports at height n. Cache hits
 are decoded and checked like downloads. Failed downloads and corrupt files
 fail the run, and a reply is stored only after it decodes as evidence, so an
 unconfirmed status is never cached.
+Parent headers are cached as hex in `{blockhash}.header`; their ordered
+`{blockhash}.txids.json` lists must reproduce the hash-verified header's merkle root.
 """
 
 from collections.abc import Callable, Sequence
@@ -19,13 +21,15 @@
 from typing import TypeVar
 from urllib.request import Request, urlopen
 
-from bitcoin.core import CTransaction, b2lx
+from bitcoin.core import CBlock, CBlockHeader, CTransaction, b2lx, lx
 
 from block_evidence import omitted_prevouts, read_transaction
 
 DEFAULT_APIS = ("https://mempool.space/api", "https://blockstream.info/api")
 PREVOUTS_DIR = Path(".cache/prevouts")
 BLOCK_HASH = re.compile(r"[0-9a-fA-F]{64}")
+TXID = re.compile(r"[0-9a-f]{64}")
+PARENT_TXIDS_LIMIT = 2 * 1024 * 1024
 T = TypeVar("T")
 
 
@@ -169,3 +173,44 @@ def load_canonical_hash(height: int, cache_dir: Path | str = PREVOUTS_DIR, fetch
     what = f"block hash for height {height}"
     return _cached(what, Path(cache_dir) / f"height-{height}.hash", lambda data: decode_block_hash(data, height),
                    fetch, lambda: _fetch(what, apis, f"block-height/{height}", 256))
+
+
+def decode_parent_header(data: bytes, block_hash: str) -> CBlockHeader:
+    """Decode an Esplora hex header and bind it to the requested parent hash."""
+    if len(data) > 256:
+        raise ValueError("oversized parent header")
+    raw = bytes.fromhex(data.decode("ascii", errors="replace").strip())
+    if len(raw) != 80:
+        raise ValueError("parent header must encode 80 bytes")
+    header = CBlockHeader.deserialize(raw)
+    if b2lx(header.GetHash()) != block_hash:
+        raise ValueError("parent header identity mismatch")
+    return header
+
+
+def decode_parent_txids(data: bytes, header: CBlockHeader) -> list[str]:
+    """Authenticate a complete, ordered txid list against the parent merkle root."""
+    if len(data) > PARENT_TXIDS_LIMIT:
+        raise ValueError("oversized parent txid list")
+    txids = json.loads(data)
+    if not isinstance(txids, list) or not txids or any(
+            not isinstance(txid, str) or not TXID.fullmatch(txid) for txid in txids):
+        raise ValueError("parent txid list must be a nonempty array of lowercase 64-hex strings")
+    # Repeated leaves can preserve a merkle root under Bitcoin's odd-leaf padding.
+    if len(set(txids)) != len(txids):
+        raise ValueError("duplicate transaction IDs in parent evidence")
+    if CBlock.build_merkle_tree_from_txids([lx(txid) for txid in txids])[-1] != header.hashMerkleRoot:
+        raise ValueError("parent transaction merkle root mismatch")
+    return txids
+
+
+def load_parent_txids(block_hash: str, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False,
+                      apis: Sequence[str] = DEFAULT_APIS) -> list[str]:
+    """Load a hash-verified parent header and its merkle-authenticated txid list."""
+    cache = Path(cache_dir)
+    header = _cached(f"parent header {block_hash}", cache / f"{block_hash}.header",
+                     lambda data: decode_parent_header(data, block_hash), fetch,
+                     lambda: _fetch(f"parent header {block_hash}", apis, f"block/{block_hash}/header", 256))
+    return _cached(f"parent txids {block_hash}", cache / f"{block_hash}.txids.json",
+                   lambda data: decode_parent_txids(data, header), fetch,
+                   lambda: _fetch(f"parent txids {block_hash}", apis, f"block/{block_hash}/txids", PARENT_TXIDS_LIMIT))

ci/sanity-check.py

@@ -23,18 +23,18 @@
 
 from block_evidence import (
     MAX_BLOCK_SIGOPS_COST, confirmed_at_or_after, establishes_rule, omitted_prevouts,
-    read_block, sigop_cost, verify_witness_commitment,
+    read_block, reuses_parent_transaction, sigop_cost, verify_witness_commitment,
 )
 from prevouts import (
-    DEFAULT_APIS, PREVOUTS_DIR, load_canonical_hash, load_confirmation, load_previous, load_transaction,
+    DEFAULT_APIS, PREVOUTS_DIR, load_canonical_hash, load_confirmation, load_parent_txids, load_previous, load_transaction,
 )
 
 DATA_PATH = Path("data/invalid-blocks.jsonl")
 BLOCKS_DIR = Path("blocks")
 REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"}
 CONTEXT_FIELDS = {
     "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex",
-    "pool", "pool_basis", "parent_kind", "missing_prevout",
+    "pool", "pool_basis", "parent_kind", "missing_prevout", "parent_txid",
 }
 OUTPOINT = re.compile(r"[0-9a-f]{64}:(?:0|[1-9][0-9]*)")
 OBSERVATION_REQUIRED = {"channel", "source", "provenance"}
@@ -47,13 +47,16 @@
 
 # Evidence paths: local = header/context only; body = complete block file;
 # sigops = body plus previous transactions; missing_parent = body plus API
-# evidence for the recorded outpoint. Rule names and reject strings must
+# evidence for the recorded outpoint; parent_txid_reuse = body plus an
+# authenticated canonical parent txid list. Rule names and reject strings must
 # match docs/schema.md.
 RULES = {
     "bad-txns-vout-toolarge": ("bad-txns-vout-toolarge", (), "body"),
     "bad-blk-sigops": ("bad-blk-sigops", (), "sigops"),
     "bad-txns-inputs-missingorspent": ("bad-txns-inputs-missingorspent", (), "body"),
     "missing_unconfirmed_parent": ("bad-txns-inputs-missingorspent", ("missing_prevout",), "missing_parent"),
+    "already_confirmed_in_parent": ("bad-txns-inputs-missingorspent",
+        ("parent_txid", "parent_kind", "coinbase_height", "coinbase_scriptsig_hex"), "parent_txid_reuse"),
     "bip34_v2_coinbase_height_mismatch": (
         "bad-cb-height", ("coinbase_height", "coinbase_scriptsig_hex"), "local"),
     "bip34_coinbase_height_mismatch": (
@@ -170,6 +173,8 @@ def check_context(record: dict[str, Any]) -> None:
             raise ValueError(f"pool_basis must be one of {sorted(POOL_BASES)}")
     elif "pool_basis" in details:
         raise ValueError("pool_basis requires pool")
+    if "parent_txid" in details:
+        hex_value(details, "parent_txid", 32)
     if "missing_prevout" in details and not (
             isinstance(details["missing_prevout"], str) and OUTPOINT.fullmatch(details["missing_prevout"])):
         raise ValueError("missing_prevout must be txid:vout in lowercase hex")
@@ -283,6 +288,16 @@ def check_local_evidence(record: dict[str, Any], header: CBlockHeader) -> None:
         raise ValueError("coinbase scriptSig must exceed 100 bytes")
 
 
+def require_canonical_parent(record: dict[str, Any], prevouts_dir: Path | str, fetch_prevouts: bool,
+                             apis: Sequence[str]) -> str:
+    """Require prev_hash to be the block the API reports at the previous height."""
+    previous_height = record["height"] - 1
+    canonical = load_canonical_hash(previous_height, prevouts_dir, fetch_prevouts, apis)
+    if canonical != record["prev_hash"]:
+        raise ValueError(f"prev_hash is not the canonical block at height {previous_height}")
+    return canonical
+
+
 def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevouts_dir: Path | str = PREVOUTS_DIR,
                            fetch_prevouts: bool = False, apis: Sequence[str] = DEFAULT_APIS) -> None:
     """Require a checked failure; observations cannot substitute for bytes."""
@@ -293,14 +308,21 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
         raise ValueError(f"{record['rule']} requires a complete block body")
     if mode == "body" and not establishes_rule(block, record["rule"]):
         raise ValueError(f"committed body does not demonstrate {record['rule']}")
+    if mode == "parent_txid_reuse":
+        context = record["context"]
+        if context["parent_kind"] != "canonical":
+            raise ValueError("already_confirmed_in_parent requires parent_kind=canonical")
+        if context["coinbase_height"] != record["height"]:
+            raise ValueError("already_confirmed_in_parent requires coinbase_height equal to height")
+        require_canonical_parent(record, prevouts_dir, fetch_prevouts, apis)
+        parent_txids = load_parent_txids(record["prev_hash"], prevouts_dir, fetch_prevouts, apis)
+        if not reuses_parent_transaction(block, parent_txids, context["parent_txid"]):
+            raise ValueError("parent_txid is not a non-coinbase transaction in both the body and its parent")
     if mode == "missing_parent":
         txid, vout = record["context"]["missing_prevout"].split(":")
         if (lx(txid), int(vout)) not in omitted_prevouts(block.vtx):
             raise ValueError("missing_prevout is not spent by the body from outside the block")
-        previous_height = record["height"] - 1
-        canonical = load_canonical_hash(previous_height, prevouts_dir, fetch_prevouts, apis)
-        if canonical != record["prev_hash"]:
-            raise ValueError(f"prev_hash is not the canonical block at height {previous_height}")
+        canonical = require_canonical_parent(record, prevouts_dir, fetch_prevouts, apis)
         parent = load_transaction(txid, prevouts_dir, fetch_prevouts, apis)
         if int(vout) >= len(parent.vout):
             raise ValueError(f"missing_prevout output {vout} does not exist in the parent transaction")
@@ -309,7 +331,7 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
             raise ValueError("parent transaction is not confirmed at this height or later in another block")
         if fetch_prevouts:
             print(f"{record['height']}: parent {txid} currently confirmed at {confirmation[0]} "
-                  f"in {confirmation[1]}; canonical block at {previous_height} is {canonical}", flush=True)
+                  f"in {confirmation[1]}; canonical block at {record['height'] - 1} is {canonical}", flush=True)
     if mode == "sigops":
         # This checker uses BIP16 + BIP141 counting, not pre-SegWit rules.
         if record["height"] < 481824:
@@ -397,7 +419,7 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--fetch-prevouts", action="store_true",
-                        help="fetch missing sigops prevouts, parent confirmations and canonical block hashes from public APIs")
+                        help="fetch missing transactions, confirmations, canonical hashes and parent txid evidence from public APIs")
     parser.add_argument("--prevouts-dir", type=Path, default=PREVOUTS_DIR, help="verified transaction cache directory")
     parser.add_argument("--api-url", action="append", help="Esplora API base URL; repeat for fallback providers")
     args = parser.parse_args()

ci/test_block_evidence.py

@@ -2,12 +2,12 @@
 
 import unittest
 
-from bitcoin.core import CBlock, COutPoint, CTransaction, CTxIn, CTxOut, CTxWitness, CTxInWitness
+from bitcoin.core import CBlock, COutPoint, CTransaction, CTxIn, CTxOut, CTxWitness, CTxInWitness, b2lx
 from bitcoin.core.script import CScript, CScriptWitness, OP_TRUE
 
 from block_evidence import (
     MAX_MONEY, confirmed_at_or_after, establishes_rule, omitted_prevouts, read_block, sha256d,
-    sigop_count, witness_sigops,
+    reuses_parent_transaction, sigop_count, witness_sigops,
 )
 
 
@@ -82,6 +82,25 @@ def test_invalid_block_serialization(self):
             with self.subTest(case=case), self.assertRaises(ValueError):
                 read_block(wire)
 
+    def test_parent_transaction_reuse_excludes_coinbases_and_absent_transactions(self):
+        """The named witness must be a non-coinbase transaction present in both blocks."""
+        coinbase = transaction()
+        reused = transaction(prev_hash=b"\x11" * 32, vout=0)
+        candidate = read_block(block(coinbase, reused))
+        txid = b2lx(candidate.vtx[1].GetTxid())
+        coinbase_id = b2lx(candidate.vtx[0].GetTxid())
+        cases = (
+            ("intersection", candidate, ["ab" * 32, txid], txid, True),
+            ("disjoint", candidate, ["ab" * 32, "cd" * 32], txid, False),
+            ("body coinbase", candidate, ["ab" * 32, coinbase_id], coinbase_id, False),
+            ("parent coinbase", candidate, [txid, "ab" * 32], txid, False),
+            ("absent from body", candidate, ["ab" * 32, "cd" * 32], "cd" * 32, False),
+            ("coinbase only", read_block(block(coinbase)), ["ab" * 32, txid], txid, False),
+        )
+        for name, body, parent, witness, expected in cases:
+            with self.subTest(case=name):
+                self.assertEqual(reuses_parent_transaction(body, parent, witness), expected)
+
 
 class SigopChecks(unittest.TestCase):
     def test_legacy_multisig_accurate_count_and_pushed_bytes(self):