diff --git a/README.md b/README.md index ea53289..0df8d3c 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This dataset covers blocks that fail those rules, including failures that can be - [`docs/notes.md`](docs/notes.md): replay behaviour and incident notes. - `blocks/{height}-{hash}.bin`: full block, when available. - `proofs/{height}-{hash}.json`: for a P2SH record without a body, the failing transaction and the block's ordered transaction IDs. +- [`data/reported-blocks.jsonl`](data/reported-blocks.jsonl): blocks reported as invalid whose failure is not established. Merge-mined recoveries generally provide a header and coinbase rather than a full Bitcoin block. @@ -27,6 +28,8 @@ Include `context` fields needed to establish the failure: BIP34 coinbase height Omit unknown optional fields. When the pool is known, give `pool` with `pool_basis`: `tag` for a coinbase tag, `address` for a payout address listed in [mining-pools](https://github.com/bitcoin-data/mining-pools), or `reported` when only a contemporaneous report names the pool. +A block whose failure is only reported goes in [`data/reported-blocks.jsonl`](data/reported-blocks.jsonl) with its sources, until the evidence turns up. + Include all available `observations`, with a source and provenance URL for each. Distinct child-chain blocks and independent observers remain separate observations. Use `merge_mining` for child-chain commitments, `p2p` for direct Bitcoin network reception, and `scrape` for website or API archives where direct reception is not established. diff --git a/ci/sanity-check.py b/ci/sanity-check.py index 71aa1f0..4a947ce 100644 --- a/ci/sanity-check.py +++ b/ci/sanity-check.py @@ -33,7 +33,9 @@ DATA_PATH = Path("data/invalid-blocks.jsonl") BLOCKS_DIR = Path("blocks") PROOFS_DIR = Path("proofs") +REPORTED_PATH = Path("data/reported-blocks.jsonl") REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"} +REPORTED_REQUIRED = {"height", "hash", "reported_failure", "sources"} CONTEXT_FIELDS = { "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex", "pool", "pool_basis", "parent_kind", "missing_prevout", "parent_txid", "failing_prevout", @@ -160,6 +162,54 @@ def string(record: dict[str, Any], name: str) -> str: return value +def checked_header(record: dict[str, Any]) -> CBlockHeader: + """Decode the 80-byte header and require it to hash to `hash` under a valid target.""" + header = CBlockHeader.deserialize(hex_value(record, "header", 80)) + calculated = b2lx(header.GetHash()) + if record["hash"] != calculated: + raise ValueError("header hash mismatch") + target = target_from_bits(header.nBits) + if not target or int(calculated, 16) > target: + raise ValueError("header does not satisfy a valid PoW target") + return header + + +def http_url(value: Any) -> bool: + """True for a string that parses as an HTTP(S) URL with a host.""" + url = urlparse(value) if isinstance(value, str) else None + return url is not None and url.scheme in ("http", "https") and bool(url.hostname) + + +def check_reported(path: Path | str, established: Set[str]) -> tuple[list[str], int]: + """Check the reported-blocks ledger: identity, sources, ordering and no overlap with admitted records.""" + problems = [] + seen = set() + last_key = None + for where, record in read_jsonl(Path(path), problems): + try: + check_fields(record, REPORTED_REQUIRED, REPORTED_REQUIRED | {"header"}) + height = integer(record, "height", 1) + hex_value(record, "hash", 32) + string(record, "reported_failure") + sources = record["sources"] + if not isinstance(sources, list) or not sources or not all(map(http_url, sources)): + raise ValueError("sources must be a nonempty array of HTTP(S) URLs") + if "header" in record: + checked_header(record) + key = (height, record["hash"]) + if last_key is not None and key < last_key: + raise ValueError("records must be ordered by height then hash") + last_key = key + if record["hash"] in seen: + raise ValueError(f"duplicate block hash {record['hash']}") + seen.add(record["hash"]) + if record["hash"] in established: + raise ValueError("reported block is already an admitted record") + except ValueError as exc: + problems.append(f"{where}: {exc}") + return problems, len(seen) + + def check_context(record: dict[str, Any]) -> None: """Validate context encoding and the fields required by the rule registry.""" details = record.get("context", {}) @@ -218,8 +268,7 @@ def check_observations(record: dict[str, Any]) -> int: hex_value(observation, "child_block_hash", 32) if "child_header" in observation: hex_value(observation, "child_header", 80) - url = urlparse(observation["provenance"]) - if url.scheme not in ("http", "https") or not url.hostname: + if not http_url(observation["provenance"]): raise ValueError("provenance must be an HTTP(S) URL") # A chain can commit the same Bitcoin parent at multiple child heights; # independent recorders can also observe the same block. Reject only @@ -374,7 +423,7 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_DIR, prevouts_dir: Path | str = PREVOUTS_DIR, fetch_prevouts: bool = False, apis: Sequence[str] = DEFAULT_APIS, - proofs_dir: Path | str = PROOFS_DIR) -> tuple[list[str], tuple[int, int, int, int, int]]: + proofs_dir: Path | str = PROOFS_DIR) -> tuple[list[str], set[str], tuple[int, int, int, int]]: problems = [] seen = set() remaining = {"block": {path.name: path for path in Path(blocks_dir).glob("*.bin")}, @@ -409,13 +458,7 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_ if block_hash in seen: raise ValueError(f"duplicate block hash {block_hash}") seen.add(block_hash) - parsed_header = CBlockHeader.deserialize(header) - calculated = b2lx(parsed_header.GetHash()) - if block_hash != calculated: - raise ValueError("header hash mismatch") - target = target_from_bits(parsed_header.nBits) - if not target or int(calculated, 16) > target: - raise ValueError("header does not satisfy a valid PoW target") + parsed_header = checked_header(record) if record["prev_hash"] != b2lx(parsed_header.hashPrevBlock): raise ValueError("prev_hash mismatch with header") if timestamp != parsed_header.nTime: @@ -447,8 +490,8 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_ for kind, paths in remaining.items(): for path in sorted(paths.values()): problems.append(f"{path}: orphan {kind} file; name must match a dataset record") - return problems, (len(seen), context_count, observation_count, - found["block"] - len(remaining["block"]), found["proof"] - len(remaining["proof"])) + return problems, seen, (context_count, observation_count, + found["block"] - len(remaining["block"]), found["proof"] - len(remaining["proof"])) def main() -> int: @@ -458,14 +501,17 @@ def main() -> int: parser.add_argument("--prevouts-dir", type=Path, default=PREVOUTS_DIR, help="verified transaction cache directory") parser.add_argument("--api-url", action="append", help="Esplora API base URL; repeat for fallback providers") args = parser.parse_args() - problems, counts = check_dataset(prevouts_dir=args.prevouts_dir, fetch_prevouts=args.fetch_prevouts, - apis=args.api_url or DEFAULT_APIS) + problems, admitted, counts = check_dataset(prevouts_dir=args.prevouts_dir, fetch_prevouts=args.fetch_prevouts, + apis=args.api_url or DEFAULT_APIS) + reported_problems, reported = check_reported(REPORTED_PATH, admitted) + problems += reported_problems if problems: print("sanity-check failed:") print("\n".join(problems)) return 1 print("sanity-check successful") - print(f" {counts[0]} blocks, {counts[1]} contexts, {counts[2]} observations, {counts[3]} block files, {counts[4]} proof files") + print(f" {len(admitted)} blocks, {counts[0]} contexts, {counts[1]} observations, {counts[2]} block files, {counts[3]} proof files") + print(f" {reported} reported blocks not admitted") return 0 diff --git a/ci/test_sanity_check.py b/ci/test_sanity_check.py index 79b6ca5..2c2f9cd 100644 --- a/ci/test_sanity_check.py +++ b/ci/test_sanity_check.py @@ -138,7 +138,7 @@ def test_sigops_requires_previous_transactions(self): self.copy_evidence(self.record) path = self.root / "sigops.jsonl" path.write_text(json.dumps(self.record) + "\n") - problems, _ = CHECK.check_dataset(path, self.root / "blocks", self.root / "empty-cache") + problems = CHECK.check_dataset(path, self.root / "blocks", self.root / "empty-cache")[0] self.assertTrue(any("missing cached previous transaction" in p for p in problems)) def test_missing_parent_requires_recorded_outpoint_and_cached_evidence(self): @@ -258,6 +258,28 @@ def test_p2sh_proof_stands_in_for_a_missing_body(self): (self.root / "proofs" / f"{body['height']}-{body['hash']}.json").write_text(json.dumps(proof)) self.assertTrue(any("not both" in p for p in self.validate([self.record, body]))) + def test_reported_ledger(self): + """The real ledger passes; an admitted hash, a header that does not hash to its record, empty sources, disorder and a duplicate are rejected.""" + rows = [json.loads(line) for line in CHECK.REPORTED_PATH.read_text().splitlines()] + established = {r["hash"] for r in self.records} + path = self.root / "reported.jsonl" + + def check(content): + path.write_text("".join(json.dumps(row) + "\n" for row in content)) + return CHECK.check_reported(path, established)[0] + + self.assertEqual(check(rows), []) + cases = ( + ("admitted hash", [dict(rows[0], hash=self.record["hash"])], "already an admitted record"), + ("wrong header", [dict(rows[0], header=self.record["header"])], "header hash mismatch"), + ("no sources", [dict(rows[0], sources=[])], "nonempty array"), + ("unsorted", [rows[1], rows[0]], "ordered by height"), + ("duplicate", [rows[0], rows[0]], "duplicate block hash"), + ) + for case, content, error in cases: + with self.subTest(case=case): + self.assertTrue(any(error in p for p in check(content))) + def test_body_matches_claimed_evidence(self): """Bind the named failure and supplied coinbase scriptSig to the available body.""" self.record = self.for_rule("bad-txns-vout-toolarge") diff --git a/data/reported-blocks.jsonl b/data/reported-blocks.jsonl new file mode 100644 index 0000000..3d7c149 --- /dev/null +++ b/data/reported-blocks.jsonl @@ -0,0 +1,10 @@ +{"height":197701,"hash":"00000000000003414be07bc5e700b1b2168ceb7259b4b9de4b583c435ebc03f2","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]} +{"height":197705,"hash":"0000000000000607179812c7bc8751fb834436bdb79f1ab6239d34b4f8cd2dda","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]} +{"height":197883,"hash":"000000000000011e6d02f6063d4a797ce42fe1f97ecf3dd640b8145f12c2304d","header":"020000005a7371583e7793a9d8e719a6586fb9c092ccd1b4935a42b765000000000000006243836741520333a1083c2dddf0904cf36ce705e0175df25aa7d13fb40f5041b2954b50383a061a3b092f03","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]} +{"height":212048,"hash":"0000000000000202a4ba1a09870c43b6c7f316a62b755cb82bffaea347c271b3","reported_failure":"unspecified: a node reported InvalidChainFound","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-12-13.html#l-334","https://pastebin.com/raw/LZxst5vD","https://bitcointalk.org/index.php?topic=1403436.msg14244002#msg14244002","https://web.archive.org/web/20220518172624/https://pastebin.com/LZxst5vD"]} +{"height":363997,"hash":"000000000000000003ae1223f4926ec86100885cfe1484dc52fd67e042a19b12","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]} +{"height":364261,"hash":"00000000000000000b6adf92bc192b3c21210f456ab21b5e46951665c74cfab2","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]} +{"height":367195,"hash":"0000000000000000116322b5f25826787b01f7a70fb322837b68dff8216cefc4","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]} +{"height":386682,"hash":"0000000000000000083cbdbb25c1607527c8f3fdb16f0d048c4439a73b501cb6","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]} +{"height":387396,"hash":"00000000000000000afc9fbe7cfe8a6b50502d509ba626beb2e2d6c15d1d3ee3","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]} +{"height":450529,"hash":"000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5","header":"0000002088d8c92a3cf343c66e3586efda20c57892687c26026c4d01000000000000000073bad4c202ec5306f8f4525f0a148bb03379275448b505a018b5bc4cc3feefcd9e928d5847cc021832ba9b71","reported_failure":"bad-blk-length: 1000023 bytes reported","sources":["https://bitco.in/forum/threads/buir-2017-01-29-statement-regarding-excessive-block-by-bitcoin-unlimited-software-29-jan-2017.1790/","https://web.archive.org/web/20170129223228id_/https://live.blockcypher.com/btc/block/000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5/","https://web.archive.org/web/20170713202211id_/https://live.blockcypher.com/btc/block/000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5/"]} diff --git a/docs/notes.md b/docs/notes.md index dc55fad..51d4a21 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -20,6 +20,15 @@ The records below attribute a pool another way, and `pool_basis` says which. 474294 and 477115 are attributed to 1Hash from the [BitcoinTalk thread](https://bitcointalk.org/index.php?topic=2041607.0) of July 2017 that discussed both blocks; their coinbases carry `/NYA/` and no pool tag. 226845, 226895 and 226912 are attributed to mmpool, Chris Double's [Bitparking merged-mining pool](https://bitcointalk.org/index.php?topic=57148.0) at mmpool.bitparking.com, from his [20 March 2013 post](https://bitcointalk.org/index.php?topic=57148.msg1646921#msg1646921) in that thread reporting three invalidated blocks that day, and the [bitcoin-dev log](https://buildingbitcoin.org/bitcoin-dev/log-2013-03-20.html) of the same day, where he reports the `block height mismatch in coinbase` rejection and names 226845; the later 367047 carries the `mmpool` tag itself. +## Reported blocks + +`data/reported-blocks.jsonl` keeps ten blocks that were reported as invalid but cannot be admitted. +Three Eligius blocks of September 2012 were reported as coinbase overpayments alongside the admitted 197438; 197883's header survives but its other transaction and fee do not, and 197701 and 197705 are known only by hash. +P2Pool's 212048 is known from a December 2012 `InvalidChainFound` report and a node-history dump, without a header or a rejection reason. +Five version-2 hashes from the BIP66 and BIP65 windows come from a March 2017 bitcoin-dev message and still lack headers. +Bitcoin Unlimited's 450529 has a recovered header and a reported size of 1000023 bytes, but no body bytes to establish it. +Issues labelled [`reported`](https://github.com/bitcoin-data/invalid-blocks/issues?q=label%3Areported) record what has already been searched for each incident and are the place to bring the missing header or body. + ## Incident notes ### 507514, 509557, 515319 and 534339 - AntPool parent-transaction reuse (2018) diff --git a/docs/schema.md b/docs/schema.md index f29e5e3..239e395 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -19,6 +19,7 @@ Hex strings are lowercase without `0x`. Bitcoin hashes use RPC/display byte order with leading zeros; `header` is the 160-character wire serialization. Full blocks, when available, are `blocks/{height}-{hash}.bin`. A P2SH record without a body carries `proofs/{height}-{hash}.json` instead. +Blocks whose failure was reported but never established are listed separately in `data/reported-blocks.jsonl`, described at the end of this document; they are not part of the dataset's admitted records. ## Required fields @@ -259,3 +260,19 @@ This evaluates one input with a library interpreter; it is not Bitcoin Core's in A record without a body may carry `proofs/{height}-{hash}.json`: an object with `transaction`, the hex of the failing transaction, and `txids`, the block's complete ordered transaction IDs. The list must reproduce the header's merkle root and contain the transaction's txid, which binds the transaction to the header without its other bodies; the input is then checked exactly as for a body. A record has a body or a proof file, not both, and a proof file for any other rule is an error. + +## Reported blocks + +`data/reported-blocks.jsonl` lists blocks that a contemporary source reported as invalid but whose failure the dataset cannot establish, usually because the header or body is lost. +They are not admitted records: nothing in the rule tables applies to them, and a body or proof file for one of them is an error until it becomes a record. +One object per line, sorted by `(height, hash)`, with the same encoding rules as the main file. + +| Field | Type | Meaning | +| --- | --- | --- | +| `height` | integer | Reported height. | +| `hash` | string | Reported block hash, the unique key; it must not appear in `data/invalid-blocks.jsonl`. | +| `header` | string | 80-byte header in hex, when recovered; it must hash to `hash` and meet its own target. | +| `reported_failure` | string | What the report claims, in a few words. | +| `sources` | array | HTTP(S) URLs of the reports. | + +CI checks these fields, the header when present, the ordering and the overlap; it does not verify the reports.