From ea9261997b2f6030a2162dc68f59b738bb10529c Mon Sep 17 00:00:00 2001 From: ahrazzle <38987266+ahrazzle@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:44:21 -0500 Subject: [PATCH 1/2] Prove truncated UTF-8 recovery and record mutation evidence --- Proofs/Main.lean | 3 ++- README.md | 2 +- SPEC.md | 16 +++++++++++++++- Spec.lean | 3 ++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Proofs/Main.lean b/Proofs/Main.lean index 5b7d6a7..4c5b962 100644 --- a/Proofs/Main.lean +++ b/Proofs/Main.lean @@ -139,7 +139,8 @@ theorem Spec.substitution_vectors : ['�', '�', '�', '�', 'A'] ∧ -- A cut-short sequence, then `<` and `A` Utf8.decodeLossy [0xF0, 0x9F, 0x98, 0x3C] = ['�', '<'] ∧ - Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] := by + Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] ∧ + Utf8.decodeLossy [0xE2, 0x82, 0x3C] = ['�', '<'] := by decide theorem Spec.utf16_encode_eq (cs : List Char) : Utf16.encode cs = Spec.encode16 cs := diff --git a/README.md b/README.md index 6c6dc30..d26fbf6 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The statements are in `Spec.lean`, the definitions they use in `Spec/Defs.lean` | `decodeLossy` gives exactly what section 3.9.6 of the Unicode Standard, *U+FFFD Substitution of Maximal Subparts*, sets out: each well-formed sequence becomes its character, and each maximal subpart of bad bytes becomes one U+FFFD. | `Spec.decodeLossy_iff` | | On well-formed bytes, `decodeLossy` reads what `decode` reads; on any other bytes it gives at least one U+FFFD. | `Spec.decodeLossy_of_decode`, `Spec.decodeLossy_replaces` | | Well-formed text before or after bad bytes comes through unchanged: no U+FFFD takes in any of it. | `Spec.decodeLossy_encode_append`, `Spec.decodeLossy_append_encode` | -| The examples of Tables 3-8 to 3-11 of the Unicode Standard, and two cut-short sequences followed by a character. | `Spec.substitution_vectors` | +| The examples of Tables 3-8 to 3-11 of the Unicode Standard, and three cut-short sequences followed by a character. | `Spec.substitution_vectors` | | The UTF-16 encoder writes each character as RFC 2781, section 2.1, sets out. | `Spec.utf16_encode_eq` | | `Utf16.decode` returns a list of characters exactly when the code units are its encoding. So every unit list it accepts has one reading, and it refuses everything else. | `Spec.utf16_decode_iff` | | Decoding what the UTF-16 encoder wrote gives the characters back. | `Spec.utf16_decode_encode` | diff --git a/SPEC.md b/SPEC.md index e6ee782..7749f54 100644 --- a/SPEC.md +++ b/SPEC.md @@ -121,7 +121,7 @@ Each property is a theorem in `Spec.lean`, under the name given. The definitions | On well-formed bytes, `decodeLossy` reads what `decode` reads. | `Spec.decodeLossy_of_decode` | | On bytes that are not well formed, `decodeLossy` gives at least one U+FFFD. | `Spec.decodeLossy_replaces` | | Well-formed text before or after bad bytes comes through unchanged. | `Spec.decodeLossy_encode_append`, `Spec.decodeLossy_append_encode` | -| The examples of section 3.9.6, Tables 3-8 to 3-11, and two cut-short sequences followed by a character. | `Spec.substitution_vectors` | +| The examples of section 3.9.6, Tables 3-8 to 3-11, and three cut-short sequences followed by a character. | `Spec.substitution_vectors` | | The UTF-16 encoder writes each character as RFC 2781, section 2.1, sets out. | `Spec.utf16_encode_eq` | | `Utf16.decode` returns `cs` exactly when the units are the encoding of `cs`. | `Spec.utf16_decode_iff` | | Decoding what the UTF-16 encoder wrote gives the characters back. | `Spec.utf16_decode_encode` | @@ -154,3 +154,17 @@ Together, these fix every output. Python's strict `utf-8` codec, on every input, and its `replace` error handler for the lossy decoder. Lean core's `String.fromUTF8?` at run time is checked against the same answers as the strict decoder. For UTF-16, Python's `utf-16-be` and `utf-16-le` codecs, strict and with `replace`, for the unit decoders and the byte decoders. Python's `utf-16` codec reads text that starts with a byte order mark. Text with no mark goes to `utf-16-be`, since Python's `utf-16` would read it in the machine's byte order, little-endian on most machines. For `toWellFormed`, the oracle is ECMAScript's `CodePointAt` rule, written out in `tests/run_tests.py`. + +## Mutation evidence limits + +The merged lossy UTF-8 slice lists eight behavior-changing mutations: +one replacement per bad byte; four separate second-byte bound mutations +for E0, ED, F0 and F4; failing to count the third byte of a truncated +four-byte sequence; allowing leads through F7; and a two-byte lead +swallowing the following byte. This accounts for eight, not five. + +The UTF-16 PR reports fourteen mutations, but its published list identifies +only thirteen when the two disjunctions are counted separately. No +fourteenth mutation or reproducible mutation runner is recorded in this +repository. The fourteen-run claim therefore remains unverified; the +ordinary differential results are recorded in `tests/TEST-RESULTS.md`. diff --git a/Spec.lean b/Spec.lean index 72b1759..8d8835e 100644 --- a/Spec.lean +++ b/Spec.lean @@ -179,7 +179,8 @@ theorem Spec.substitution_vectors : ['�', '�', '�', '�', 'A'] ∧ -- A cut-short sequence, then `<` and `A` Utf8.decodeLossy [0xF0, 0x9F, 0x98, 0x3C] = ['�', '<'] ∧ - Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] := sorry + Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] ∧ + Utf8.decodeLossy [0xE2, 0x82, 0x3C] = ['�', '<'] := sorry /-! ## UTF-16 code units -/ From ca29613641d61d61bf70e5a08bba4965a38b7ca7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 06:12:17 +0000 Subject: [PATCH 2/2] Fix planted-bug evidence and drop wrong SPEC.md mutation section Remove the incorrect "Mutation evidence limits" section that claimed the fourteenth UTF-16 planted bug was unverifiable. Add tests/planted_bugs.py and tests/PLANTED-BUGS.md from the review-followups record; the runner catches all 14 UTF-16 mutations (28 of 30 total, two marked equivalent). Update substitution_vectors docstrings for three cut-short cases. Co-authored-by: ahraz --- Proofs/Main.lean | 2 +- SPEC.md | 14 --- Spec.lean | 5 +- tests/PLANTED-BUGS.md | 260 ++++++++++++++++++++++++++++++++++++++++++ tests/planted_bugs.py | 250 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 513 insertions(+), 18 deletions(-) create mode 100644 tests/PLANTED-BUGS.md create mode 100755 tests/planted_bugs.py diff --git a/Proofs/Main.lean b/Proofs/Main.lean index 4c5b962..2af762a 100644 --- a/Proofs/Main.lean +++ b/Proofs/Main.lean @@ -137,7 +137,7 @@ theorem Spec.substitution_vectors : -- Table 3-11, truncated sequences Utf8.decodeLossy [0xE1, 0x80, 0xE2, 0xF0, 0x91, 0x92, 0xF1, 0xBF, 0x41] = ['�', '�', '�', '�', 'A'] ∧ - -- A cut-short sequence, then `<` and `A` + -- Three cut-short sequences, then `<`, `A`, or `<` again Utf8.decodeLossy [0xF0, 0x9F, 0x98, 0x3C] = ['�', '<'] ∧ Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] ∧ Utf8.decodeLossy [0xE2, 0x82, 0x3C] = ['�', '<'] := by diff --git a/SPEC.md b/SPEC.md index 7749f54..c3c29f5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -154,17 +154,3 @@ Together, these fix every output. Python's strict `utf-8` codec, on every input, and its `replace` error handler for the lossy decoder. Lean core's `String.fromUTF8?` at run time is checked against the same answers as the strict decoder. For UTF-16, Python's `utf-16-be` and `utf-16-le` codecs, strict and with `replace`, for the unit decoders and the byte decoders. Python's `utf-16` codec reads text that starts with a byte order mark. Text with no mark goes to `utf-16-be`, since Python's `utf-16` would read it in the machine's byte order, little-endian on most machines. For `toWellFormed`, the oracle is ECMAScript's `CodePointAt` rule, written out in `tests/run_tests.py`. - -## Mutation evidence limits - -The merged lossy UTF-8 slice lists eight behavior-changing mutations: -one replacement per bad byte; four separate second-byte bound mutations -for E0, ED, F0 and F4; failing to count the third byte of a truncated -four-byte sequence; allowing leads through F7; and a two-byte lead -swallowing the following byte. This accounts for eight, not five. - -The UTF-16 PR reports fourteen mutations, but its published list identifies -only thirteen when the two disjunctions are counted separately. No -fourteenth mutation or reproducible mutation runner is recorded in this -repository. The fourteen-run claim therefore remains unverified; the -ordinary differential results are recorded in `tests/TEST-RESULTS.md`. diff --git a/Spec.lean b/Spec.lean index 8d8835e..2954937 100644 --- a/Spec.lean +++ b/Spec.lean @@ -162,8 +162,7 @@ theorem Spec.decodeLossy_append_encode (bs : List UInt8) (cs : List Char) : Utf8.decodeLossy (bs ++ Utf8.encode cs) = Utf8.decodeLossy bs ++ cs := sorry /-- The examples of section 3.9.6 of the Unicode Standard, Tables 3-8 - to 3-11, and two cases where replacing too many bytes would swallow - the next character. -/ + to 3-11, and three cut-short sequences followed by a character. -/ theorem Spec.substitution_vectors : -- Table 3-8, non-shortest forms Utf8.decodeLossy [0xC0, 0xAF, 0xE0, 0x80, 0xBF, 0xF0, 0x81, 0x82, 0x41] = @@ -177,7 +176,7 @@ theorem Spec.substitution_vectors : -- Table 3-11, truncated sequences Utf8.decodeLossy [0xE1, 0x80, 0xE2, 0xF0, 0x91, 0x92, 0xF1, 0xBF, 0x41] = ['�', '�', '�', '�', 'A'] ∧ - -- A cut-short sequence, then `<` and `A` + -- Three cut-short sequences, then `<`, `A`, or `<` again Utf8.decodeLossy [0xF0, 0x9F, 0x98, 0x3C] = ['�', '<'] ∧ Utf8.decodeLossy [0xE2, 0x41, 0x42] = ['�', 'A', 'B'] ∧ Utf8.decodeLossy [0xE2, 0x82, 0x3C] = ['�', '<'] := sorry diff --git a/tests/PLANTED-BUGS.md b/tests/PLANTED-BUGS.md new file mode 100644 index 0000000..c4d1c0d --- /dev/null +++ b/tests/PLANTED-BUGS.md @@ -0,0 +1,260 @@ +# Planted bugs + +Written by `python3 tests/planted_bugs.py`. Each row is one small change to the +code under proof, made in a copy. Before any change, the copy's tool builds, the +tests report 0 failures, and `lake build Proofs` succeeds. With each change, the +tests ran as `python3 tests/run_tests.py --count 20000` runs them, on 2,415,209 +inputs from their recorded seed, counted as `tests/TEST-RESULTS.md` counts them, +against the changed tool. A bug is caught when they report failures. The last +column says whether `lake build Proofs` still succeeds with the change. + +| Bug | File | Failures | Proofs | +|-----|------|----------|--------| +| Three-byte overlong forms read | `Utf8/Codec.lean` | 4833 | fail to build | +| Surrogates read | `Utf8/Codec.lean` | 5902 | fail to build | +| Values past U+10FFFF read | `Utf8/Codec.lean` | 920 | fail to build | +| Leads 0xC0 and 0xC1 read | `Utf8/Codec.lean` | 37877 | fail to build | +| 0xC0 read as a continuation byte | `Utf8/Codec.lean` | 19175 | fail to build | +| U+0800 written in two bytes | `Utf8/Codec.lean` | 250 | fail to build | +| Leads 0xF5 to 0xF7 start four-byte sequences | `Utf8/Codec.lean` | 0, equivalent: with a lead of 0xF5 to 0xF7 the value is at least 5 * 0x40000 = 0x140000, past 0x10FFFF, so the four-byte branch refuses it. `decodeChar` gives `none` on the same inputs as before, and the encoder is unchanged. | fail to build | +| One U+FFFD for each bad byte | `Utf8/Lossy.lean` | 261672 | fail to build | +| Second byte after 0xE0 not narrowed | `Utf8/Lossy.lean` | 8854 | fail to build | +| Second byte after 0xED not narrowed | `Utf8/Lossy.lean` | 9526 | fail to build | +| Second byte after 0xF0 not narrowed | `Utf8/Lossy.lean` | 4731 | fail to build | +| Second byte after 0xF4 not narrowed | `Utf8/Lossy.lean` | 13595 | fail to build | +| Third byte of a cut-short four-byte sequence not counted | `Utf8/Lossy.lean` | 18788 | fail to build | +| Leads 0xF5 to 0xF7 take continuation bytes | `Utf8/Lossy.lean` | 54410 | fail to build | +| Leads 0xC2 to 0xDF and 0xF5 to 0xFF swallow the next byte | `Utf8/Lossy.lean` | 696466 | fail to build | +| A bad last byte given its own branch | `Utf8/Lossy.lean` | 0, equivalent: when `rest` is empty, `badLength b0 []` is 1 for every `b0`, so the old code also gives U+FFFD and drops the one byte, leaving nothing. `lossyStep` gives the same pair on every input. The change was meant to drop a lone lead byte at the end, but as written it gives the same answer. | fail to build | +| High surrogate range misses 0xDBFF | `Utf16/Codec.lean` | 3943 | fail to build | +| Low surrogate range misses 0xDC00 | `Utf16/Codec.lean` | 8679 | fail to build | +| U+10000 written as one unit | `Utf16/Codec.lean` | 969 | fail to build | +| Pair arithmetic off by one | `Utf16/Codec.lean` | 30786 | fail to build | +| Lone low surrogate accepted | `Utf16/Codec.lean` | 11129 | fail to build | +| High surrogate paired with any unit | `Utf16/Codec.lean` | 3569 | fail to build | +| Lossy decoding swallows the unit after a high surrogate | `Utf16/Codec.lean` | 7677 | fail to build | +| Lossy decoding drops a final high surrogate | `Utf16/Codec.lean` | 10627 | fail to build | +| Two U+FFFD for a high surrogate and an odd byte | `Utf16/Bytes.lean` | 764 | fail to build | +| Big-endian mark kept as text | `Utf16/Bytes.lean` | 998 | fail to build | +| Text with no mark read little-endian | `Utf16/Bytes.lean` | 73811 | fail to build | +| UTF-16LE written big-endian | `Utf16/Bytes.lean` | 2642 | fail to build | +| Odd byte ignored by the strict byte decoders | `Utf16/Bytes.lean` | 8192 | fail to build | +| Little-endian mark read big-endian | `Utf16/Bytes.lean` | 1441 | fail to build | + +## The changes + +**Three-byte overlong forms read** in `Utf8/Codec.lean`: + +``` +- if n < 0x800 ∨ (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none ++ if (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none +``` + +**Surrogates read** in `Utf8/Codec.lean`: + +``` +- if n < 0x800 ∨ (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none ++ if n < 0x800 then none +``` + +**Values past U+10FFFF read** in `Utf8/Codec.lean`: + +``` +- if n < 0x10000 ∨ 0x10FFFF < n then none ++ if n < 0x10000 then none +``` + +**Leads 0xC0 and 0xC1 read** in `Utf8/Codec.lean`: + +``` +- else if x < 0xC2 then none ++ else if x < 0xC0 then none +``` + +**0xC0 read as a continuation byte** in `Utf8/Codec.lean`: + +``` +- if 0x80 ≤ b.toNat ∧ b.toNat ≤ 0xBF then ++ if 0x80 ≤ b.toNat ∧ b.toNat ≤ 0xC0 then +``` + +**U+0800 written in two bytes** in `Utf8/Codec.lean`: + +``` +- else if n < 0x800 then +- [(0xC0 ++ else if n ≤ 0x800 then ++ [(0xC0 +``` + +**Leads 0xF5 to 0xF7 start four-byte sequences** in `Utf8/Codec.lean`: + +``` +- else if x < 0xF5 then ++ else if x < 0xF8 then +``` + +**One U+FFFD for each bad byte** in `Utf8/Lossy.lean`: + +``` +- (b0 :: rest).drop (badLength b0 rest) ++ (b0 :: rest).drop 1 +``` + +**Second byte after 0xE0 not narrowed** in `Utf8/Lossy.lean`: + +``` +- if x = 0xE0 then 0xA0 ≤ y ++ if x = 0xE0 then 0x80 ≤ y +``` + +**Second byte after 0xED not narrowed** in `Utf8/Lossy.lean`: + +``` +- else if x = 0xED then 0x80 ≤ y && y ≤ 0x9F ++ else if x = 0xED then 0x80 ≤ y && y ≤ 0xBF +``` + +**Second byte after 0xF0 not narrowed** in `Utf8/Lossy.lean`: + +``` +- else if x = 0xF0 then 0x90 ≤ y ++ else if x = 0xF0 then 0x80 ≤ y +``` + +**Second byte after 0xF4 not narrowed** in `Utf8/Lossy.lean`: + +``` +- else if x = 0xF4 then 0x80 ≤ y && y ≤ 0x8F ++ else if x = 0xF4 then 0x80 ≤ y && y ≤ 0xBF +``` + +**Third byte of a cut-short four-byte sequence not counted** in `Utf8/Lossy.lean`: + +``` +- then 3 else 2 ++ then 2 else 2 +``` + +**Leads 0xF5 to 0xF7 take continuation bytes** in `Utf8/Lossy.lean`: + +``` +- if 0xE0 ≤ x ∧ x ≤ 0xF4 then ++ if 0xE0 ≤ x ∧ x ≤ 0xF7 then +``` + +**Leads 0xC2 to 0xDF and 0xF5 to 0xFF swallow the next byte** in `Utf8/Lossy.lean`: + +``` +- else 1 +- +- /-- One step ++ else if 0xC2 ≤ x ∧ rest ≠ [] then 2 else 1 ++ ++ /-- One step +``` + +**A bad last byte given its own branch** in `Utf8/Lossy.lean`: + +``` +- | none => (replacement, (b0 :: rest).drop (badLength b0 rest)) ++ | none => (if rest = [] then (replacement, []) else (replacement, (b0 :: rest).drop (badLength b0 rest))) +``` + +**High surrogate range misses 0xDBFF** in `Utf16/Codec.lean`: + +``` +- u.toNat ≤ 0xDBFF ++ u.toNat < 0xDBFF +``` + +**Low surrogate range misses 0xDC00** in `Utf16/Codec.lean`: + +``` +- 0xDC00 ≤ u.toNat ++ 0xDC01 ≤ u.toNat +``` + +**U+10000 written as one unit** in `Utf16/Codec.lean`: + +``` +- if n < 0x10000 then [n.toUInt16] ++ if n ≤ 0x10000 then [n.toUInt16] +``` + +**Pair arithmetic off by one** in `Utf16/Codec.lean`: + +``` +- Char.ofNat (0x10000 + (hi.toNat ++ Char.ofNat (0xFFFF + (hi.toNat +``` + +**Lone low surrogate accepted** in `Utf16/Codec.lean`: + +``` +- else if isLow u then none ++ else if false then none +``` + +**High surrogate paired with any unit** in `Utf16/Codec.lean`: + +``` +- if isLow v then (decode rest').map (pair u v :: ·) else none ++ (decode rest').map (pair u v :: ·) +``` + +**Lossy decoding swallows the unit after a high surrogate** in `Utf16/Codec.lean`: + +``` +- else replacement :: decodeLossy (v :: rest') ++ else replacement :: decodeLossy rest' +``` + +**Lossy decoding drops a final high surrogate** in `Utf16/Codec.lean`: + +``` +- | [] => [replacement] ++ | [] => [] +``` + +**Two U+FFFD for a high surrogate and an odd byte** in `Utf16/Bytes.lean`: + +``` +- if us.getLast?.any isHigh then decodeLossy us else decodeLossy us ++ [replacement] ++ decodeLossy us ++ [replacement] +``` + +**Big-endian mark kept as text** in `Utf16/Bytes.lean`: + +``` +- if a = 0xFE ∧ b = 0xFF then decodeBE rest ++ if a = 0xFE ∧ b = 0xFF then decodeBE (a :: b :: rest) +``` + +**Text with no mark read little-endian** in `Utf16/Bytes.lean`: + +``` +- else decodeBE (a :: b :: rest) ++ else decodeLE (a :: b :: rest) +``` + +**UTF-16LE written big-endian** in `Utf16/Bytes.lean`: + +``` +- [(u.toNat % 0x100).toUInt8, (u.toNat / 0x100).toUInt8] ++ [(u.toNat / 0x100).toUInt8, (u.toNat % 0x100).toUInt8] +``` + +**Odd byte ignored by the strict byte decoders** in `Utf16/Bytes.lean`: + +``` +- | (_, some _) => none ++ | (us, some _) => decode us +``` + +**Little-endian mark read big-endian** in `Utf16/Bytes.lean`: + +``` +- else if a = 0xFF ∧ b = 0xFE then decodeLE rest ++ else if a = 0xFF ∧ b = 0xFE then decodeBE rest +``` diff --git a/tests/planted_bugs.py b/tests/planted_bugs.py new file mode 100755 index 0000000..2878ce5 --- /dev/null +++ b/tests/planted_bugs.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Plant each bug below in a copy of the code, and check the tests catch it. + +A planted bug is a small change to the code under proof, of a kind a +person could make by mistake. The script copies the project into a +temporary directory. There, before any change, it checks that the tool +builds, that the tests report 0 failures, and that the proofs build. +Then, for each bug, it makes the change, builds the command-line tool, +and runs the differential tests as `tests/run_tests.py --count N` runs +them. The tests catch the bug when they report failures. + +A change that alters an answer of the code under proof also breaks the +proofs, since the code no longer meets its statements, and a full +`lake build` then fails. So the tests run against the tool built on its +own, as `tests/run_tests.py` builds it. After the tests, the script +builds the proofs against the changed code and records whether they +still build. + +Apart from tests/PLANTED-BUGS.md, which it rewrites, the script never +changes the project; each bug is planted in the copy. It exits with 1 +if a bug is not caught and is not marked as equivalent, and with an +error if a test run gives no clear result. An equivalent change gives +the same answer on every input, so no test can catch it; its entry says +why. + +Usage: python3 tests/planted_bugs.py [--count N] +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +EXE = "utf8" # the lake target of the command-line tool +COUNT = 20_000 + +# (name, file, text to replace, replacement, why it is equivalent or None) +BUGS = [ + # The strict UTF-8 codec. + ("Three-byte overlong forms read", "Utf8/Codec.lean", + "if n < 0x800 ∨ (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none", + "if (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none", None), + ("Surrogates read", "Utf8/Codec.lean", + "if n < 0x800 ∨ (0xD800 ≤ n ∧ n ≤ 0xDFFF) then none", + "if n < 0x800 then none", None), + ("Values past U+10FFFF read", "Utf8/Codec.lean", + "if n < 0x10000 ∨ 0x10FFFF < n then none", + "if n < 0x10000 then none", None), + ("Leads 0xC0 and 0xC1 read", "Utf8/Codec.lean", + "else if x < 0xC2 then none", + "else if x < 0xC0 then none", None), + ("0xC0 read as a continuation byte", "Utf8/Codec.lean", + "if 0x80 ≤ b.toNat ∧ b.toNat ≤ 0xBF then", + "if 0x80 ≤ b.toNat ∧ b.toNat ≤ 0xC0 then", None), + ("U+0800 written in two bytes", "Utf8/Codec.lean", + " else if n < 0x800 then\n [(0xC0", + " else if n ≤ 0x800 then\n [(0xC0", None), + ("Leads 0xF5 to 0xF7 start four-byte sequences", "Utf8/Codec.lean", + "else if x < 0xF5 then", + "else if x < 0xF8 then", + "with a lead of 0xF5 to 0xF7 the value is at least 5 * 0x40000 = 0x140000, " + "past 0x10FFFF, so the four-byte branch refuses it. `decodeChar` gives `none` " + "on the same inputs as before, and the encoder is unchanged."), + # The lossy UTF-8 decoder. + ("One U+FFFD for each bad byte", "Utf8/Lossy.lean", + "(b0 :: rest).drop (badLength b0 rest)", + "(b0 :: rest).drop 1", None), + ("Second byte after 0xE0 not narrowed", "Utf8/Lossy.lean", + "if x = 0xE0 then 0xA0 ≤ y", + "if x = 0xE0 then 0x80 ≤ y", None), + ("Second byte after 0xED not narrowed", "Utf8/Lossy.lean", + "else if x = 0xED then 0x80 ≤ y && y ≤ 0x9F", + "else if x = 0xED then 0x80 ≤ y && y ≤ 0xBF", None), + ("Second byte after 0xF0 not narrowed", "Utf8/Lossy.lean", + "else if x = 0xF0 then 0x90 ≤ y", + "else if x = 0xF0 then 0x80 ≤ y", None), + ("Second byte after 0xF4 not narrowed", "Utf8/Lossy.lean", + "else if x = 0xF4 then 0x80 ≤ y && y ≤ 0x8F", + "else if x = 0xF4 then 0x80 ≤ y && y ≤ 0xBF", None), + ("Third byte of a cut-short four-byte sequence not counted", "Utf8/Lossy.lean", + "then 3 else 2", + "then 2 else 2", None), + ("Leads 0xF5 to 0xF7 take continuation bytes", "Utf8/Lossy.lean", + "if 0xE0 ≤ x ∧ x ≤ 0xF4 then", + "if 0xE0 ≤ x ∧ x ≤ 0xF7 then", None), + ("Leads 0xC2 to 0xDF and 0xF5 to 0xFF swallow the next byte", "Utf8/Lossy.lean", + " else 1\n\n/-- One step", + " else if 0xC2 ≤ x ∧ rest ≠ [] then 2 else 1\n\n/-- One step", None), + ("A bad last byte given its own branch", "Utf8/Lossy.lean", + "| none => (replacement, (b0 :: rest).drop (badLength b0 rest))", + "| none => (if rest = [] then (replacement, []) else " + "(replacement, (b0 :: rest).drop (badLength b0 rest)))", + "when `rest` is empty, `badLength b0 []` is 1 for every `b0`, so the old code also " + "gives U+FFFD and drops the one byte, leaving nothing. `lossyStep` gives the same " + "pair on every input. The change was meant to drop a lone lead byte at the end, " + "but as written it gives the same answer."), + # UTF-16 code units. + ("High surrogate range misses 0xDBFF", "Utf16/Codec.lean", + "u.toNat ≤ 0xDBFF", + "u.toNat < 0xDBFF", None), + ("Low surrogate range misses 0xDC00", "Utf16/Codec.lean", + "0xDC00 ≤ u.toNat", + "0xDC01 ≤ u.toNat", None), + ("U+10000 written as one unit", "Utf16/Codec.lean", + "if n < 0x10000 then [n.toUInt16]", + "if n ≤ 0x10000 then [n.toUInt16]", None), + ("Pair arithmetic off by one", "Utf16/Codec.lean", + "Char.ofNat (0x10000 + (hi.toNat", + "Char.ofNat (0xFFFF + (hi.toNat", None), + ("Lone low surrogate accepted", "Utf16/Codec.lean", + " else if isLow u then none", + " else if false then none", None), + ("High surrogate paired with any unit", "Utf16/Codec.lean", + "if isLow v then (decode rest').map (pair u v :: ·) else none", + "(decode rest').map (pair u v :: ·)", None), + ("Lossy decoding swallows the unit after a high surrogate", "Utf16/Codec.lean", + "else replacement :: decodeLossy (v :: rest')", + "else replacement :: decodeLossy rest'", None), + ("Lossy decoding drops a final high surrogate", "Utf16/Codec.lean", + " | [] => [replacement]", + " | [] => []", None), + # UTF-16 as bytes. + ("Two U+FFFD for a high surrogate and an odd byte", "Utf16/Bytes.lean", + "if us.getLast?.any isHigh then decodeLossy us else decodeLossy us ++ [replacement]", + "decodeLossy us ++ [replacement]", None), + ("Big-endian mark kept as text", "Utf16/Bytes.lean", + "if a = 0xFE ∧ b = 0xFF then decodeBE rest", + "if a = 0xFE ∧ b = 0xFF then decodeBE (a :: b :: rest)", None), + ("Text with no mark read little-endian", "Utf16/Bytes.lean", + " else decodeBE (a :: b :: rest)", + " else decodeLE (a :: b :: rest)", None), + ("UTF-16LE written big-endian", "Utf16/Bytes.lean", + "[(u.toNat % 0x100).toUInt8, (u.toNat / 0x100).toUInt8]", + "[(u.toNat / 0x100).toUInt8, (u.toNat % 0x100).toUInt8]", None), + ("Odd byte ignored by the strict byte decoders", "Utf16/Bytes.lean", + " | (_, some _) => none", + " | (us, some _) => decode us", None), + ("Little-endian mark read big-endian", "Utf16/Bytes.lean", + " else if a = 0xFF ∧ b = 0xFE then decodeLE rest", + " else if a = 0xFF ∧ b = 0xFE then decodeBE rest", None), +] + + +def failures(out): + m = re.search(r"(\d+) failures?", out) + return int(m.group(1)) if m else None + + +def build(work, target): + return subprocess.run(["lake", "build", target], cwd=work, capture_output=True, text=True) + + +def test(work, name, count): + """The number of failures the tests report in `work`. Exits when the + run gives no failure count, or when its exit status disagrees with + the count.""" + run = subprocess.run([sys.executable, "tests/run_tests.py", "--count", str(count)], + cwd=work, capture_output=True, text=True) + n = failures(run.stdout + run.stderr) + if n is None: + sys.exit(f"{name}: the tests gave no failure count\n{(run.stdout + run.stderr)[-2000:]}") + if (run.returncode == 0) != (n == 0): + sys.exit(f"{name}: the tests exited with {run.returncode} but report {n} failures\n" + f"{(run.stdout + run.stderr)[-2000:]}") + return n + + +def run(count): + rows = [] + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "project" + shutil.copytree(ROOT, work, ignore=shutil.ignore_patterns(".git")) + # The control: with no change, everything must pass. + tool = build(work, EXE) + if tool.returncode != 0: + sys.exit(f"with no change, the tool does not build\n{tool.stdout[-2000:]}") + if test(work, "with no change", count) != 0: + sys.exit("with no change, the tests report failures") + proofs = build(work, "Proofs") + if proofs.returncode != 0: + sys.exit(f"with no change, the proofs do not build\n{proofs.stdout[-2000:]}") + m = re.search(r"^- Inputs: (\d+)$", (work / "tests" / "TEST-RESULTS.md").read_text(), + re.M) + if not m: + sys.exit("with no change, the test record gives no count of inputs") + inputs = int(m.group(1)) + print(f"with no change: the tool builds, 0 failures on {inputs} inputs, the proofs build", + flush=True) + for name, rel, old, new, equivalent in BUGS: + path = work / rel + original = (ROOT / rel).read_text() + if original.count(old) != 1: + sys.exit(f"{name}: the text to replace is not in {rel} exactly once") + path.write_text(original.replace(old, new)) + tool = build(work, EXE) + if tool.returncode != 0: + path.write_text(original) + sys.exit(f"{name}: the tool does not build with this change\n{tool.stdout[-2000:]}") + n = test(work, name, count) + broke = build(work, "Proofs").returncode != 0 + path.write_text(original) + rows.append((name, rel, old, new, equivalent, n, broke)) + print(f"{name}: {n} failures; proofs {'fail to build' if broke else 'build'}", + flush=True) + return rows, inputs + + +def write(rows, count, inputs): + intro = ( + "Written by `python3 tests/planted_bugs.py`. Each row is one small change to the " + "code under proof, made in a copy. Before any change, the copy's tool builds, the " + "tests report 0 failures, and `lake build Proofs` succeeds. With each change, the " + f"tests ran as `python3 tests/run_tests.py --count {count}` runs them, on {inputs:,} " + "inputs from their recorded seed, counted as `tests/TEST-RESULTS.md` counts them, " + "against the changed tool. A bug is caught when they report failures. The last " + "column says whether `lake build Proofs` still succeeds with the change." + ) + lines = ["# Planted bugs", "", *textwrap.wrap(intro, 80), "", + "| Bug | File | Failures | Proofs |", + "|-----|------|----------|--------|"] + for name, rel, _, _, equivalent, n, broke in rows: + cell = str(n) if n else ("0, equivalent: " + equivalent if equivalent else "0, NOT CAUGHT") + lines.append(f"| {name} | `{rel}` | {cell} | {'fail to build' if broke else 'build'} |") + lines += ["", "## The changes", ""] + for name, rel, old, new, _, _, _ in rows: + lines += [f"**{name}** in `{rel}`:", "", "```", "- " + old.replace("\n", "\n- "), + "+ " + new.replace("\n", "\n+ "), "```", ""] + (ROOT / "tests" / "PLANTED-BUGS.md").write_text("\n".join(lines)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--count", type=int, default=COUNT) + args = ap.parse_args() + rows, inputs = run(args.count) + write(rows, args.count, inputs) + missed = [r[0] for r in rows if not r[5] and not r[4]] + caught = sum(1 for r in rows if r[5]) + print(f"{caught} of {len(rows)} planted bugs caught; results in tests/PLANTED-BUGS.md") + if missed: + sys.exit("not caught: " + ", ".join(missed)) + + +if __name__ == "__main__": + main()