From e1bb4c1e0f31ee19fb99adecee4f7dae7f8f18b5 Mon Sep 17 00:00:00 2001 From: Orda Date: Wed, 11 Nov 2026 13:35:58 -0600 Subject: [PATCH 1/4] Add initial configuration in bar.toml --- bar.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 bar.toml diff --git a/bar.toml b/bar.toml new file mode 100644 index 0000000..444c92a --- /dev/null +++ b/bar.toml @@ -0,0 +1,15 @@ +[project] +name = "http" +status = "in-progress" +page = "README.md" +spec_authors = ["ahrazzle"] +entry_points = ["Main.lean"] + +[spec] +files = ["SPEC.md", "Http/Types.lean", "Http/Spec/Defs.lean", "Http/Spec.lean"] +code = ["Http.Types", "Http.Framing", "Http"] + +[tests] +results = "tests/TEST-RESULTS.md" +command = ["python3", "tests/run_tests.py"] +files = ["tests/run_tests.py", "tests/planted_bugs.py", "tests/PLANTED-BUGS.md"] From a4481889a496b60958a88db24f0143d4876f65b7 Mon Sep 17 00:00:00 2001 From: Orda Date: Wed, 11 Nov 2026 13:38:56 -0600 Subject: [PATCH 2/4] Add HTTP/1.1 Message Framing and Chunked Transfer Coding spec This document outlines the rules for parsing and framing HTTP/1.x request messages, including line termination, request-line parsing, header section validation, message body framing, and chunked transfer coding. --- SPEC.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 SPEC.md diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..608fcae --- /dev/null +++ b/SPEC.md @@ -0,0 +1,22 @@ +# HTTP/1.1 Message Framing and Chunked Transfer Coding (`Http`) + +This specification defines the wire-level rules for parsing and framing an HTTP/1.x request message from a byte stream under RFC 9112 (HTTP/1.1) and RFC 9110 (HTTP Semantics), taking the strict reading wherever lenient parsers permit request-boundary disagreement. + +## Scope + +1. **Line Termination (`splitCRLF`)**: Every start-line, header field line, chunk-size line, chunk-data record, and trailer section MUST end with `CRLF` (`0x0D 0x0A`). Any bare `CR` (`0x0D` not followed by `0x0A`) or bare `LF` (`0x0A` not preceded by `0x0D`) inside a line is rejected (`none`). +2. **Request-Line (`parseRequestLine`, RFC 9112 Section 3)**: Must split on single `SP` (`0x20`) into three components `method SP request-target SP HTTP-version`. Runs of multiple spaces, `HTAB`, or control characters are rejected. `method` is `1*tchar`; `request-target` is `1*VCHAR`; `HTTP-version` is `HTTP/1.1` or `HTTP/1.0`. +3. **Header Section (`parseHeaders`, RFC 9112 Section 5)**: + - Obsolete line folding (`obs-fold`, lines beginning with `SP` or `HTAB`, Section 5.2) is rejected. + - Whitespace between `field-name` and `:` (Section 5.1) is rejected; `field-name` must satisfy `1*tchar`. + - Leading and trailing optional whitespace (`OWS`: `SP` / `HTAB`) around `field-value` is stripped; interior bytes must satisfy RFC 9110 Section 5.5 (`field-vchar` or `OWS`, no `NUL` or control octets). +4. **Message Body Framing (`determineFraming`, RFC 9112 Sections 6.1-6.3)**: + - **Combined `Transfer-Encoding` and `Content-Length` (Sections 6.1 / 6.3)**: Unconditionally rejected to prevent `CL.TE` and `TE.CL` boundary ambiguity. + - **`Transfer-Encoding` in `HTTP/1.0` (Section 6.1, EEF-CVE-2026-94194)**: Unconditionally rejected. + - **Transfer-Coding List (Sections 6.1 / 6.3, EEF-CVE-2026-94194)**: Tokenized across comma-separated values; empty elements, non-token characters, repeated `chunked`, or non-final `chunked` (`Transfer-Encoding: chunked, gzip`) are rejected. Only `[chunked]` (case-insensitively) selects chunked framing in this slice. + - **`Content-Length` (Section 6.2 / RFC 9110 Section 8.6)**: Duplicate `Content-Length` header fields are rejected; a single `Content-Length` value must match `1*DIGIT` (`< 2^64`, no `+`, `-`, or comma list). + - **Default**: Requests with neither `Transfer-Encoding` nor `Content-Length` have a zero-byte body (`fixed 0`). +5. **Chunked Transfer Coding (`decodeChunked`, RFC 9112 Section 7.1)**: + - `chunk-size` is `1*HEXDIG` (`< 2^64`). Optional chunk extensions after `;` must contain only `OWS` and `field-vchar`. + - Every non-zero chunk payload of `sz` bytes MUST be followed immediately by `0x0D 0x0A` (`CRLF`); non-CRLF terminators (CVE-2025-23167 class) are rejected. + - The terminating `0` chunk is followed by an optional RFC 9112 Section 7.1.2 trailer section (`*( field-line CRLF )`) and final `CRLF`. From e8920cc5a5af399c1ecf9226981835aa7b6edcc6 Mon Sep 17 00:00:00 2001 From: Orda Date: Wed, 11 Nov 2026 13:40:07 -0600 Subject: [PATCH 3/4] Enhance README with library details and theorems Expanded README to include detailed library specifications, planned theorems, and unproven aspects. --- README.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2da4a73..955e541 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ -# http -lean HTTP request-line and header parser (RFC 9112) +# http: HTTP/1.1 request framing and chunked decoding, proved + +This library specifies, frames, and serializes HTTP/1.x request messages in Lean 4 following RFC 9112 (Sections 2, 3, 5, 6, and 7.1), targeting machine-checked theorems for prefix-free stream framing and rejection of HTTP request-boundary ambiguities (`CL.TE`, `TE.CL`, `TE.TE`, `HTTP/1.0` `Transfer-Encoding`, and malformed chunk terminators). + +Status: in progress. The specification has not yet been approved by an independent reviewer on the org's list, so this is not yet verified. + +## Planned and Drafted Theorems (`Http.Spec` / `Http.Proofs`) + +| Claim | Theorem | +| :--- | :--- | +| `splitCRLF` splits at the earliest `CRLF` with no bare `CR` or `LF` in the line. | `Http.Spec.splitCRLF_sound` | +| `splitCRLF` is prefix-free when trailing pipelined bytes are appended. | `Http.Spec.splitCRLF_prefix_free` | +| Decimal formatting of numbers below `2^64` parses back to the same number. | `Http.Spec.parseDec_printDec` | +| Hexadecimal formatting of numbers below `2^64` parses back to the same number. | `Http.Spec.parseHex_printHex` | +| Body framing resolution assigns at most one framing mode to any header list. | `Http.Spec.determineFraming_disjoint` | +| Requests carrying both `Transfer-Encoding` and `Content-Length` are always rejected. | `Http.Spec.determineFraming_rejects_cl_and_te` | +| `HTTP/1.0` requests carrying `Transfer-Encoding` are always rejected (EEF-CVE-2026-94194). | `Http.Spec.determineFraming_rejects_http10_te` | +| Chunked framing requires `HTTP/1.1` and zero `Content-Length` headers. | `Http.Spec.determineFraming_rejects_nonfinal_chunked` | +| `encodeChunked` round-trips through `decodeChunked` with empty remainder. | `Http.Spec.decodeChunked_encodeChunked` | +| `decodeChunked` is prefix-free across pipelined TCP byte streams. | `Http.Spec.decodeChunked_prefix_free` | +| Canonical fixed-length request serialization round-trips through `parseRequest`. | `Http.Spec.parseRequest_serializeFixed` | +| Canonical chunked request serialization round-trips through `parseRequest`. | `Http.Spec.parseRequest_serializeChunked` | +| `parseRequest` is prefix-free when pipelined bytes follow on the TCP stream. | `Http.Spec.parseRequest_prefix_free` | +| Multi-message TCP streams decompose uniquely under `StreamFrames`. | `Http.Spec.parseRequest_stream_unique` | +| Normative RFC 9112 / CVE boundary-ambiguity vectors are rejected. | `Http.Spec.reject_smuggling_vectors` | + +## What is not proved + +- Hex CLI decoding/encoding in `Main.lean` (covered by differential tests). +- HTTP response framing (`1xx`, `204`, `304`, and `HEAD` bodyless response rules in RFC 9112 Section 6.3). +- Content-coding decompression (`gzip`, `deflate`, `br`) and TLS / socket I/O. From 72b5eef46bc9a050f0cc053a672bd8d1d45204d9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 06:49:22 +0000 Subject: [PATCH 4/4] Prove request framing and chunked decoding, and align the spec with the request-head rules. The challenge statements now include the strengthened chunked-framing theorem, the CI workflow is the verified bar, and the differential tests compare http-tool with a second transcription of SPEC.md. Co-authored-by: ahraz --- .github/workflows/bar.yml | 21 + .gitignore | 2 + Http.lean | 4 + Http/Bytes.lean | 443 +++++++++++++++ Http/Framing.lean | 117 ++++ Http/Lemmas.lean | 1123 +++++++++++++++++++++++++++++++++++++ Http/Parse.lean | 391 +++++++++++++ Http/Proofs.lean | 103 ++++ Http/Spec.lean | 100 ++++ Http/Spec/Defs.lean | 31 + Http/Types.lean | 43 ++ Main.lean | 18 + README.md | 101 +++- SPEC.md | 103 +++- bar.toml | 20 +- comparator.json | 28 + lake-manifest.json | 6 + lakefile.toml | 19 + lean-toolchain | 1 + tests/TEST-RESULTS.md | 6 + tests/run_tests.py | 672 ++++++++++++++++++++++ 21 files changed, 3304 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/bar.yml create mode 100644 .gitignore create mode 100644 Http.lean create mode 100644 Http/Bytes.lean create mode 100644 Http/Framing.lean create mode 100644 Http/Lemmas.lean create mode 100644 Http/Parse.lean create mode 100644 Http/Proofs.lean create mode 100644 Http/Spec.lean create mode 100644 Http/Spec/Defs.lean create mode 100644 Http/Types.lean create mode 100644 Main.lean create mode 100644 comparator.json create mode 100644 lake-manifest.json create mode 100644 lakefile.toml create mode 100644 lean-toolchain create mode 100644 tests/TEST-RESULTS.md create mode 100644 tests/run_tests.py diff --git a/.github/workflows/bar.yml b/.github/workflows/bar.yml new file mode 100644 index 0000000..32457ed --- /dev/null +++ b/.github/workflows/bar.yml @@ -0,0 +1,21 @@ +name: verified bar + +on: + push: + pull_request: + # Rerun when the spec reviewer approves, or an approval is dismissed. + pull_request_review: + types: [submitted, dismissed] + +jobs: + bar: + # pull-requests: read lets the check confirm the spec approval on GitHub. + permissions: + contents: read + pull-requests: read + # Both lines pin the same commit of Proven-Commons/mission: bar v2, as + # proposed in its pull request #2. Move both to a release tag once + # mission publishes one. + uses: Proven-Commons/mission/.github/workflows/verified-bar.yml@6ba85c27abaaf07905c9316aa96dc29f7b5847a2 + with: + bar-ref: 6ba85c27abaaf07905c9316aa96dc29f7b5847a2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae73679 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.lake/ +__pycache__/ diff --git a/Http.lean b/Http.lean new file mode 100644 index 0000000..65fd024 --- /dev/null +++ b/Http.lean @@ -0,0 +1,4 @@ +import Http.Types +import Http.Bytes +import Http.Framing +import Http.Parse diff --git a/Http/Bytes.lean b/Http/Bytes.lean new file mode 100644 index 0000000..d1a2fae --- /dev/null +++ b/Http/Bytes.lean @@ -0,0 +1,443 @@ +import Http.Types + +namespace Http + +def lowerByte (b : UInt8) : UInt8 := + if 65 ≤ b && b ≤ 90 then b + 32 else b + +def mapBytes (f : UInt8 → UInt8) : List UInt8 → List UInt8 + | [] => [] + | b :: bs => f b :: mapBytes f bs + +def lowerBytes (bs : List UInt8) : List UInt8 := + mapBytes lowerByte bs + +def isOWS (b : UInt8) : Bool := + b == 32 || b == 9 + +def dropWhileBytes (p : UInt8 → Bool) : List UInt8 → List UInt8 + | [] => [] + | b :: bs => if p b then dropWhileBytes p bs else b :: bs + +def revBytes : List UInt8 → List UInt8 → List UInt8 + | [], acc => acc + | b :: bs, acc => revBytes bs (b :: acc) + +def trimOWS (bs : List UInt8) : List UInt8 := + revBytes (dropWhileBytes isOWS (revBytes (dropWhileBytes isOWS bs) [])) [] + +def allBytes (p : UInt8 → Bool) : List UInt8 → Bool + | [] => true + | b :: bs => p b && allBytes p bs + +def anyBytes (p : UInt8 → Bool) : List UInt8 → Bool + | [] => false + | b :: bs => p b || anyBytes p bs + +def takeBytes : Nat → List UInt8 → List UInt8 + | 0, _ => [] + | _, [] => [] + | n + 1, b :: bs => b :: takeBytes n bs + +def dropBytes : Nat → List UInt8 → List UInt8 + | 0, bs => bs + | _, [] => [] + | n + 1, _ :: bs => dropBytes n bs + +def takeWhileBytes (p : UInt8 → Bool) : List UInt8 → List UInt8 + | [] => [] + | b :: bs => if p b then b :: takeWhileBytes p bs else [] + +def splitByte (sep : UInt8) : List UInt8 → List (List UInt8) + | [] => [[]] + | b :: bs => + let parts := splitByte sep bs + if b == sep then + [] :: parts + else + match parts with + | [] => [[b]] + | p :: ps => (b :: p) :: ps + +/-- Split at the first CRLF. A bare CR or a bare LF makes the split fail, +and neither byte is kept in the line. -/ +def splitCRLF : List UInt8 → Option (List UInt8 × List UInt8) + | [] => none + | c :: bs => + if c = 0x0D then + match bs with + | [] => none + | d :: rest => if d = 0x0A then some ([], rest) else none + else if c = 0x0A then + none + else + match splitCRLF bs with + | none => none + | some (line, rest) => some (c :: line, rest) + +def noBareLineBreak (line : List UInt8) : Prop := + ∀ b, b ∈ line → b ≠ 0x0D ∧ b ≠ 0x0A + +theorem splitCRLF_of_line (line rest : List UInt8) (h : noBareLineBreak line) : + splitCRLF (line ++ [0x0D, 0x0A] ++ rest) = some (line, rest) := by + induction line with + | nil => + simp [splitCRLF] + | cons c line ih => + have hc : c ≠ 0x0D ∧ c ≠ 0x0A := h c (by simp) + have hline : noBareLineBreak line := fun b hb => h b (by simp [hb]) + have ih' := ih hline + rcases hc with ⟨hcr, hlf⟩ + have ih'' : splitCRLF (line ++ 0x0D :: 0x0A :: rest) = some (line, rest) := by + simpa [List.append_assoc] using ih' + simp [splitCRLF, hcr, hlf, ih''] + +theorem splitCRLF_sound_left (bs line rest : List UInt8) + (h : splitCRLF bs = some (line, rest)) : + noBareLineBreak line ∧ bs = line ++ [0x0D, 0x0A] ++ rest := by + induction bs generalizing line rest with + | nil => + simp [splitCRLF] at h + | cons c bs ih => + by_cases hcr : c = 0x0D + · subst hcr + cases bs with + | nil => + simp [splitCRLF] at h + | cons d ds => + by_cases hlf : d = 0x0A + · subst hlf + simp [splitCRLF] at h + rcases h with ⟨rfl, rfl⟩ + simp [noBareLineBreak] + · simp [splitCRLF, hlf] at h + · by_cases hlf : c = 0x0A + · simp [splitCRLF, hlf] at h + · simp only [splitCRLF, hcr, hlf] at h + cases hsp : splitCRLF bs with + | none => + simp [hsp] at h + | some pair => + obtain ⟨line', rest'⟩ := pair + simp [hsp] at h + rcases h with ⟨rfl, rfl⟩ + obtain ⟨hn, heq⟩ := ih line' rest' hsp + refine ⟨?_, ?_⟩ + · intro b hb + simp at hb + cases hb with + | inl hb => + subst hb + exact ⟨hcr, hlf⟩ + | inr hb => + exact hn b hb + · simp [heq] + +theorem splitCRLF_sound (bs line rest : List UInt8) : + splitCRLF bs = some (line, rest) ↔ + noBareLineBreak line ∧ bs = line ++ [0x0D, 0x0A] ++ rest := by + constructor + · exact splitCRLF_sound_left bs line rest + · intro h + rcases h with ⟨hn, heq⟩ + subst heq + exact splitCRLF_of_line line rest hn + +theorem splitCRLF_prefix_free (bs line rest extra : List UInt8) + (h : splitCRLF bs = some (line, rest)) : + splitCRLF (bs ++ extra) = some (line, rest ++ extra) := by + obtain ⟨hn, heq⟩ := splitCRLF_sound_left bs line rest h + have h' := splitCRLF_of_line line (rest ++ extra) hn + simpa [heq, List.append_assoc] using h' + +def decVal (b : UInt8) : Option Nat := + if 48 ≤ b.toNat ∧ b.toNat ≤ 57 then some (b.toNat - 48) else none + +def hexVal (b : UInt8) : Option Nat := + if 48 ≤ b.toNat ∧ b.toNat ≤ 57 then some (b.toNat - 48) + else if 97 ≤ b.toNat ∧ b.toNat ≤ 102 then some (b.toNat - 87) + else if 65 ≤ b.toNat ∧ b.toNat ≤ 70 then some (b.toNat - 55) + else none + +def parseAcc (base : Nat) (val : UInt8 → Option Nat) (acc : Nat) : List UInt8 → Option Nat + | [] => if acc ≤ maxUInt64 then some acc else none + | b :: bs => + match val b with + | none => none + | some d => + let n := acc * base + d + if n ≤ maxUInt64 then parseAcc base val n bs else none + +def parseDec (ds : List UInt8) : Option Nat := + match ds with + | [] => none + | _ => parseAcc 10 decVal 0 ds + +def parseHex (ds : List UInt8) : Option Nat := + match ds with + | [] => none + | _ => parseAcc 16 hexVal 0 ds + +def natToDecChars (n : Nat) : List UInt8 := + if n < 10 then [UInt8.ofNat (48 + n)] + else natToDecChars (n / 10) ++ [UInt8.ofNat (48 + n % 10)] +termination_by n +decreasing_by + apply Nat.div_lt_self + · omega + · decide + +def hexDigit (d : Nat) : UInt8 := + if d < 10 then UInt8.ofNat (48 + d) else UInt8.ofNat (87 + d) + +def natToHexLower (n : Nat) : List UInt8 := + let digit := hexDigit (n % 16) + if n < 16 then [digit] + else natToHexLower (n / 16) ++ [digit] +termination_by n +decreasing_by + apply Nat.div_lt_self + · omega + · decide + +def decDigits (n : Nat) : Nat := + if n < 10 then 1 else decDigits (n / 10) + 1 +termination_by n +decreasing_by + apply Nat.div_lt_self + · omega + · decide + +def hexDigits (n : Nat) : Nat := + if n < 16 then 1 else hexDigits (n / 16) + 1 +termination_by n +decreasing_by + apply Nat.div_lt_self + · omega + · decide + +theorem toNat_ofNat_small (n : Nat) (h : n < 256) : (UInt8.ofNat n).toNat = n := by + simp [UInt8.ofNat, UInt8.toNat, Nat.mod_eq_of_lt h] + +theorem decVal_digit (d : Nat) (h : d < 10) : + decVal (UInt8.ofNat (48 + d)) = some d := by + have hlt : 48 + d < 256 := by omega + unfold decVal + rw [toNat_ofNat_small _ hlt] + have hlo : 48 ≤ 48 + d := Nat.le_add_right _ _ + have hhi : 48 + d ≤ 57 := by omega + simp only [hlo, hhi, and_self, ite_true] + exact congrArg some (Nat.add_sub_cancel_left 48 d) + +theorem hexVal_digit (d : Nat) (h : d < 16) : + hexVal (hexDigit d) = some d := by + unfold hexDigit hexVal + by_cases h10 : d < 10 + · have hlt : 48 + d < 256 := by omega + simp only [h10, ite_true] + rw [toNat_ofNat_small _ hlt] + have hlo : 48 ≤ 48 + d := Nat.le_add_right _ _ + have hhi : 48 + d ≤ 57 := by omega + simp only [hlo, hhi, and_self, ite_true] + exact congrArg some (Nat.add_sub_cancel_left 48 d) + · have hlt : 87 + d < 256 := by omega + simp only [h10, ite_false] + rw [toNat_ofNat_small _ hlt] + have hnot : ¬(48 ≤ 87 + d ∧ 87 + d ≤ 57) := by omega + have hlo : 97 ≤ 87 + d := by omega + have hhi : 87 + d ≤ 102 := by omega + simp only [hnot, ite_false, hlo, hhi, and_self, ite_true] + exact congrArg some (Nat.add_sub_cancel_left 87 d) + +theorem parseAcc_append (base : Nat) (val : UInt8 → Option Nat) (acc : Nat) + (xs ys : List UInt8) (hacc : acc ≤ maxUInt64) : + parseAcc base val acc (xs ++ ys) = + match parseAcc base val acc xs with + | none => none + | some n => parseAcc base val n ys := by + induction xs generalizing acc with + | nil => + simp [parseAcc, hacc] + | cons b xs ih => + simp only [List.cons_append, parseAcc] + cases hval : val b with + | none => + simp [hval] + | some d => + simp only [hval] + by_cases hn : acc * base + d ≤ maxUInt64 + · simp only [hn, ite_true] + have hacc' : acc * base + d ≤ maxUInt64 := hn + simpa using ih (acc * base + d) hacc' + · simp [hn] + +theorem parseAcc_dec (acc n : Nat) (hacc : acc ≤ maxUInt64) + (h : acc * 10 ^ decDigits n + n ≤ maxUInt64) : + parseAcc 10 decVal acc (natToDecChars n) = some (acc * 10 ^ decDigits n + n) := by + induction n using Nat.strongRecOn generalizing acc with + | ind n ih => + by_cases h10 : n < 10 + · have hchars : natToDecChars n = [UInt8.ofNat (48 + n)] := by + rw [natToDecChars] + split + · rfl + · contradiction + have hdig : decDigits n = 1 := by + rw [decDigits] + split + · rfl + · contradiction + rw [hdig, Nat.pow_one] at h + rw [hchars, hdig, Nat.pow_one] + unfold parseAcc + rw [decVal_digit n h10] + simp only [h, ite_true] + unfold parseAcc + simp only [h, ite_true] + · have hdiv : n / 10 < n := Nat.div_lt_self (by omega) (by decide) + have hdigits : decDigits n = decDigits (n / 10) + 1 := by + rw [decDigits] + split + · contradiction + · rfl + have hpow : 10 ^ decDigits n = 10 ^ decDigits (n / 10) * 10 := by + rw [hdigits, Nat.pow_succ, Nat.mul_comm] + have hassoc : acc * (10 ^ decDigits (n / 10) * 10) = acc * 10 ^ decDigits (n / 10) * 10 := by + simp [Nat.mul_assoc] + have hbound : acc * 10 ^ decDigits (n / 10) + n / 10 ≤ maxUInt64 := by + have hle : acc * 10 ^ decDigits (n / 10) + n / 10 ≤ acc * 10 ^ decDigits n + n := by + rw [hpow, hassoc] + have hmul : acc * 10 ^ decDigits (n / 10) ≤ acc * 10 ^ decDigits (n / 10) * 10 := + Nat.le_mul_of_pos_right _ (by decide) + have hdivle : n / 10 ≤ n := Nat.div_le_self _ _ + omega + exact Nat.le_trans hle h + have ih' := ih (n / 10) hdiv acc hacc hbound + have hdigit : n % 10 < 10 := Nat.mod_lt _ (by decide) + have hd := decVal_digit (n % 10) hdigit + have hstep : (acc * 10 ^ decDigits (n / 10) + n / 10) * 10 + n % 10 = + acc * 10 ^ decDigits n + n := by + have hdivmod : n / 10 * 10 + n % 10 = n := by + simpa [Nat.mul_comm] using Nat.div_add_mod n 10 + have hmul : (acc * 10 ^ decDigits (n / 10) + n / 10) * 10 = + acc * 10 ^ decDigits (n / 10) * 10 + n / 10 * 10 := by + rw [Nat.add_mul] + rw [hmul, Nat.add_assoc, hdivmod, hpow, hassoc] + have hsum : (acc * 10 ^ decDigits (n / 10) + n / 10) * 10 + n % 10 ≤ maxUInt64 := by + rw [hstep]; exact h + have hchars : natToDecChars n = + natToDecChars (n / 10) ++ [UInt8.ofNat (48 + n % 10)] := by + rw [natToDecChars] + split + · contradiction + · rfl + have happ := parseAcc_append 10 decVal acc (natToDecChars (n / 10)) + [UInt8.ofNat (48 + n % 10)] hacc + rw [hchars, happ, ih'] + unfold parseAcc + rw [hd] + simp only [hsum, ite_true, hstep] + unfold parseAcc + simp only [h, ite_true] + +theorem natToDecChars_ne_nil (n : Nat) : natToDecChars n ≠ [] := by + rw [natToDecChars] + split + · simp + · simp [List.append_eq_nil_iff] + +theorem parseDec_printDec (n : Nat) (h : n ≤ maxUInt64) : + parseDec (natToDecChars n) = some n := by + have hparse := parseAcc_dec 0 n (by decide) (by simpa using h) + have hne := natToDecChars_ne_nil n + have hzero : 0 * 10 ^ decDigits n + n = n := by simp + rw [hzero] at hparse + cases hchars : natToDecChars n with + | nil => exact absurd hchars hne + | cons b bs => + rw [hchars] at hparse + unfold parseDec + simp only [hchars] + exact hparse + +theorem parseAcc_hex (acc n : Nat) (hacc : acc ≤ maxUInt64) + (h : acc * 16 ^ hexDigits n + n ≤ maxUInt64) : + parseAcc 16 hexVal acc (natToHexLower n) = some (acc * 16 ^ hexDigits n + n) := by + induction n using Nat.strongRecOn generalizing acc with + | ind n ih => + by_cases h16 : n < 16 + · have hmod : n % 16 = n := Nat.mod_eq_of_lt h16 + have hchars : natToHexLower n = [hexDigit n] := by + rw [natToHexLower] + simp [h16, hmod] + have hdig : hexDigits n = 1 := by + rw [hexDigits] + simp [h16] + rw [hdig, Nat.pow_one] at h + rw [hchars, hdig, Nat.pow_one] + unfold parseAcc + rw [hexVal_digit n h16] + simp only [h, ite_true] + unfold parseAcc + simp only [h, ite_true] + · have hdiv : n / 16 < n := Nat.div_lt_self (by omega) (by decide) + have hdigits : hexDigits n = hexDigits (n / 16) + 1 := by + rw [hexDigits] + simp [h16] + have hpow : 16 ^ hexDigits n = 16 ^ hexDigits (n / 16) * 16 := by + rw [hdigits, Nat.pow_succ, Nat.mul_comm] + have hassoc : acc * (16 ^ hexDigits (n / 16) * 16) = acc * 16 ^ hexDigits (n / 16) * 16 := by + simp [Nat.mul_assoc] + have hbound : acc * 16 ^ hexDigits (n / 16) + n / 16 ≤ maxUInt64 := by + have hle : acc * 16 ^ hexDigits (n / 16) + n / 16 ≤ acc * 16 ^ hexDigits n + n := by + rw [hpow, hassoc] + have hmul : acc * 16 ^ hexDigits (n / 16) ≤ acc * 16 ^ hexDigits (n / 16) * 16 := + Nat.le_mul_of_pos_right _ (by decide) + have hdivle : n / 16 ≤ n := Nat.div_le_self _ _ + omega + exact Nat.le_trans hle h + have ih' := ih (n / 16) hdiv acc hacc hbound + have hdigit : n % 16 < 16 := Nat.mod_lt _ (by decide) + have hd := hexVal_digit (n % 16) hdigit + have hstep : (acc * 16 ^ hexDigits (n / 16) + n / 16) * 16 + n % 16 = + acc * 16 ^ hexDigits n + n := by + have hdivmod : n / 16 * 16 + n % 16 = n := by + simpa [Nat.mul_comm] using Nat.div_add_mod n 16 + have hmul : (acc * 16 ^ hexDigits (n / 16) + n / 16) * 16 = + acc * 16 ^ hexDigits (n / 16) * 16 + n / 16 * 16 := by + rw [Nat.add_mul] + rw [hmul, Nat.add_assoc, hdivmod, hpow, hassoc] + have hsum : (acc * 16 ^ hexDigits (n / 16) + n / 16) * 16 + n % 16 ≤ maxUInt64 := by + rw [hstep]; exact h + have hchars : natToHexLower n = natToHexLower (n / 16) ++ [hexDigit (n % 16)] := by + rw [natToHexLower] + simp [h16] + have happ := parseAcc_append 16 hexVal acc (natToHexLower (n / 16)) [hexDigit (n % 16)] hacc + rw [hchars, happ, ih'] + unfold parseAcc + rw [hd] + simp only [hsum, ite_true, hstep] + unfold parseAcc + simp only [h, ite_true] + +theorem natToHexLower_ne_nil (n : Nat) : natToHexLower n ≠ [] := by + rw [natToHexLower] + by_cases h16 : n < 16 + · simp [h16] + · simp [h16, List.append_eq_nil_iff] + +theorem parseHex_printHex (n : Nat) (h : n ≤ maxUInt64) : + parseHex (natToHexLower n) = some n := by + have hparse := parseAcc_hex 0 n (by decide) (by simpa using h) + have hne := natToHexLower_ne_nil n + have hzero : 0 * 16 ^ hexDigits n + n = n := by simp + rw [hzero] at hparse + cases hchars : natToHexLower n with + | nil => exact absurd hchars hne + | cons b bs => + rw [hchars] at hparse + unfold parseHex + simp only [hchars] + exact hparse + +end Http diff --git a/Http/Framing.lean b/Http/Framing.lean new file mode 100644 index 0000000..dd74786 --- /dev/null +++ b/Http/Framing.lean @@ -0,0 +1,117 @@ +import Http.Types +import Http.Bytes + +namespace Http + +def isDigit (b : UInt8) : Bool := + 48 ≤ b && b ≤ 57 + +def isHex (b : UInt8) : Bool := + isDigit b || (97 ≤ b && b ≤ 102) || (65 ≤ b && b ≤ 70) + +def isAlpha (b : UInt8) : Bool := + (65 ≤ b && b ≤ 90) || (97 ≤ b && b ≤ 122) + +def isTchar (b : UInt8) : Bool := + isAlpha b || isDigit b || b == 33 || b == 35 || b == 36 || b == 37 || b == 38 || + b == 39 || b == 42 || b == 43 || b == 45 || b == 46 || b == 94 || b == 95 || + b == 96 || b == 124 || b == 126 + +def isVChar (b : UInt8) : Bool := + 33 ≤ b && b ≤ 126 + +/-- A transfer-coding token: no parameters, no whitespace. -/ +def bareToken : List UInt8 → Bool + | [] => false + | bs => allBytes isTchar bs + +def joinComma : List (List UInt8) → List UInt8 + | [] => [] + | [v] => v + | v :: vs => v ++ [44] ++ joinComma vs + +def teValues : List Header → List (List UInt8) + | [] => [] + | h :: hs => + if lowerBytes h.name == teNameLower then h.value :: teValues hs else teValues hs + +def mapTrim : List (List UInt8) → List (List UInt8) + | [] => [] + | b :: bs => trimOWS b :: mapTrim bs + +def commaElements (bs : List UInt8) : List (List UInt8) := + mapTrim (splitByte 44 bs) + +def dropEmpty : List (List UInt8) → List (List UInt8) + | [] => [] + | b :: bs => if b.isEmpty then dropEmpty bs else b :: dropEmpty bs + +def allBare : List (List UInt8) → Bool + | [] => true + | b :: bs => bareToken b && allBare bs + +def mapLower : List (List UInt8) → List (List UInt8) + | [] => [] + | b :: bs => lowerBytes b :: mapLower bs + +/-- Transfer-coding list after combining field lines, dropping empty +elements, and rejecting a parameter or any other non-token. No +Transfer-Encoding field gives `some []`. -/ +def normalizedCodings (hdrs : List Header) : Option (List (List UInt8)) := + let vals := teValues hdrs + if vals.isEmpty then + some [] + else + let nonempty := dropEmpty (commaElements (joinComma vals)) + if allBare nonempty then + some (mapLower nonempty) + else + none + +def clHeaders : List Header → List Header + | [] => [] + | h :: hs => + if lowerBytes h.name == clNameLower then h :: clHeaders hs else clHeaders hs + +def digits : List UInt8 → Bool + | [] => false + | bs => allBytes isDigit bs + +inductive ClState where + | absent + | ok (n : Nat) + | bad + deriving DecidableEq, Repr + +def clState (hdrs : List Header) : ClState := + match clHeaders hdrs with + | [] => .absent + | [h] => + if digits h.value then + match parseDec h.value with + | some n => .ok n + | none => .bad + else + .bad + | _ => .bad + +/-- Body framing from the version and the header list. Chunked framing is +returned only for HTTP/1.1 whose normalized Transfer-Encoding list is exactly +`[chunked]` and which has no Content-Length. -/ +def determineFraming (v : Version) (hdrs : List Header) : Option Framing := + match normalizedCodings hdrs with + | none => none + | some cs => + match clState hdrs with + | .bad => none + | .ok n => + if cs.isEmpty && (teValues hdrs).isEmpty then some (.fixed n) else none + | .absent => + match cs with + | [] => + if (teValues hdrs).isEmpty then some (.fixed 0) else none + | [c] => + if v = .http11 && c = chunkedToken then some .chunked else none + | _ => none + +end Http diff --git a/Http/Lemmas.lean b/Http/Lemmas.lean new file mode 100644 index 0000000..9c923f9 --- /dev/null +++ b/Http/Lemmas.lean @@ -0,0 +1,1123 @@ +import Http.Parse +import Http.Spec.Defs + +namespace Http + +theorem beq_eq_false {a b : List UInt8} (h : (a == b) = false) : a ≠ b := by + intro heq + simp [heq] at h + +theorem beq_eq_true {a b : List UInt8} (h : (a == b) = true) : a = b := by + simpa using h + +theorem clHeaders_absent (hdrs : List Header) : + clHeaders hdrs = [] ↔ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower := by + induction hdrs with + | nil => simp [clHeaders] + | cons h hs ih => + constructor + · intro hempty hdr hmem + cases hcmp : lowerBytes h.name == clNameLower + · simp [clHeaders, hcmp] at hempty + simp at hmem + cases hmem with + | inl he => + subst he + exact beq_eq_false hcmp + | inr hmem => + exact (ih.mp hempty) hdr hmem + · simp [clHeaders, hcmp] at hempty + · intro hall + cases hcmp : lowerBytes h.name == clNameLower + · simp [clHeaders, hcmp] + exact ih.mpr (fun hdr hmem => hall hdr (by simp [List.mem_cons, hmem])) + · exact absurd (beq_eq_true hcmp) (hall h (by simp)) + +theorem mem_clHeaders (hdrs : List Header) (h : Header) : + h ∈ clHeaders hdrs → h ∈ hdrs ∧ lowerBytes h.name = clNameLower := by + induction hdrs with + | nil => simp [clHeaders] + | cons a as ih => + cases hcmp : lowerBytes a.name == clNameLower + · simp [clHeaders, hcmp] + intro hm + rcases ih hm with ⟨hm, heq⟩ + exact ⟨by simp [List.mem_cons, hm], heq⟩ + · simp [clHeaders, hcmp] + intro hm + rcases hm with rfl | hm + · exact ⟨by simp, beq_eq_true hcmp⟩ + · rcases ih hm with ⟨hm, heq⟩ + exact ⟨by simp [List.mem_cons, hm], heq⟩ + +theorem teValues_absent (hdrs : List Header) : + teValues hdrs = [] ↔ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ teNameLower := by + induction hdrs with + | nil => simp [teValues] + | cons h hs ih => + constructor + · intro hempty hdr hmem + cases hcmp : lowerBytes h.name == teNameLower + · simp [teValues, hcmp] at hempty + simp [List.mem_cons] at hmem + cases hmem with + | inl he => + subst he + exact beq_eq_false hcmp + | inr hmem => + exact (ih.mp hempty) hdr hmem + · simp [teValues, hcmp] at hempty + · intro hall + cases hcmp : lowerBytes h.name == teNameLower + · simp [teValues, hcmp] + exact ih.mpr (fun hdr hmem => hall hdr (by simp [List.mem_cons, hmem])) + · exact absurd (beq_eq_true hcmp) (hall h (by simp)) + +theorem clState_absent (hdrs : List Header) : + clState hdrs = .absent ↔ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower := by + unfold clState + cases hcs : clHeaders hdrs with + | nil => + simp + exact (clHeaders_absent hdrs).mp hcs + | cons h hs => + constructor + · intro habs + cases hs with + | nil => + cases hdig : digits h.value + · simp [hdig] at habs + · cases hdec : parseDec h.value <;> simp [hdig, hdec] at habs + | cons => simp [hcs] at habs + · intro hall + have hm : h ∈ clHeaders hdrs := by simp [hcs] + rcases mem_clHeaders hdrs h hm with ⟨hmem, heq⟩ + exact absurd heq (hall h hmem) + +theorem no_te (hdrs : List Header) (hte : ∃ hdr, hdr ∈ hdrs ∧ lowerBytes hdr.name = teNameLower) : + teValues hdrs ≠ [] := by + intro hempty + rcases hte with ⟨hdr, hm, heq⟩ + exact (teValues_absent hdrs).mp hempty hdr hm heq + +theorem no_cl (hdrs : List Header) (hcl : ∃ hdr, hdr ∈ hdrs ∧ lowerBytes hdr.name = clNameLower) : + clState hdrs ≠ .absent := by + intro habs + rcases hcl with ⟨hdr, hm, heq⟩ + exact (clState_absent hdrs).mp habs hdr hm heq + +theorem determineFraming_rejects_nonfinal_chunked (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ (∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower) ∧ + normalizedCodings hdrs = some [chunkedToken] := by + unfold determineFraming at h + cases hcodes : normalizedCodings hdrs with + | none => simp [hcodes] at h + | some cs => + cases hcl : clState hdrs with + | bad => simp [hcodes, hcl] at h + | ok n => simp [hcodes, hcl] at h + | absent => + cases cs with + | nil => simp [hcodes, hcl] at h + | cons c rest => + cases rest with + | nil => + cases hv : decide (v = .http11) <;> cases hc : decide (c = chunkedToken) <;> + simp [hcodes, hcl, hv, hc] at h + refine ⟨?_, (clState_absent hdrs).mp hcl, ?_⟩ + · simpa using hv + · simp [hcodes] + simpa using hc + | cons => simp [hcodes, hcl] at h + +theorem chunked_requires_http11_without_cl (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower := by + rcases determineFraming_rejects_nonfinal_chunked v hdrs h with ⟨hv, hcl, _⟩ + exact ⟨hv, hcl⟩ + +theorem determineFraming_disjoint (v : Version) (hdrs : List Header) (f1 f2 : Framing) + (h1 : determineFraming v hdrs = some f1) (h2 : determineFraming v hdrs = some f2) : + f1 = f2 := by + exact Option.some.inj (h1.symm.trans h2) + +theorem determineFraming_rejects_cl_and_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hcl : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = clNameLower) : + determineFraming v hdrs = none := by + have hte' := no_te hdrs hte + have hcl' := no_cl hdrs hcl + unfold determineFraming + cases hcodes : normalizedCodings hdrs with + | none => rfl + | some cs => + cases hst : clState hdrs with + | bad => rfl + | ok n => simp [hst, hte'] + | absent => exact absurd hst hcl' + +theorem determineFraming_rejects_http10_te (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) : + determineFraming .http10 hdrs = none := by + have hte' := no_te hdrs hte + unfold determineFraming + cases hcodes : normalizedCodings hdrs with + | none => rfl + | some cs => + cases hst : clState hdrs with + | bad => rfl + | ok n => simp [hst, hte'] + | absent => + cases cs with + | nil => simp [hst, hte'] + | cons c rest => + cases rest <;> simp + +theorem determineFraming_rejects_bad_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hbad : normalizedCodings hdrs ≠ some [chunkedToken]) : + determineFraming v hdrs = none := by + have hte' := no_te hdrs hte + unfold determineFraming + cases hcodes : normalizedCodings hdrs with + | none => rfl + | some cs => + have hcs : cs ≠ [chunkedToken] := by + intro heq + exact hbad (by simp [hcodes, heq]) + cases hst : clState hdrs with + | bad => rfl + | ok n => simp [hst, hte'] + | absent => + cases cs with + | nil => simp [hst, hte'] + | cons c rest => + cases rest with + | cons => simp + | nil => + have hc : c ≠ chunkedToken := by + intro hc + exact hcs (by simp [hc]) + simp [hc] + +theorem takeBytes_append (bs rest : List UInt8) : + takeBytes bs.length (bs ++ rest) = bs := by + induction bs with + | nil => simp [takeBytes] + | cons b bs ih => simp [takeBytes, List.length_cons, ih] + +theorem dropBytes_append (bs rest : List UInt8) : + dropBytes bs.length (bs ++ rest) = rest := by + induction bs with + | nil => simp [dropBytes] + | cons b bs ih => simp [dropBytes, List.length_cons, ih] + +theorem allBytes_append (p : UInt8 → Bool) (xs ys : List UInt8) : + allBytes p (xs ++ ys) = (allBytes p xs && allBytes p ys) := by + induction xs with + | nil => simp [allBytes] + | cons x xs ih => simp [allBytes, ih, Bool.and_assoc] + +theorem hexDigit_ok (d : Nat) (hd : d < 16) : + isHex (hexDigit d) = true ∧ hexDigit d ≠ 13 ∧ hexDigit d ≠ 10 := by + revert hd + match d with + | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 => + intro _ + decide + | n + 16 => + intro h + omega + +theorem allBytes_hex (n : Nat) : allBytes isHex (natToHexLower n) = true := by + induction n using Nat.strongRecOn with + | ind n ih => + by_cases h16 : n < 16 + · have hmod : n % 16 = n := Nat.mod_eq_of_lt h16 + simp [natToHexLower, h16, hmod, allBytes, (hexDigit_ok n h16).1] + · have hdiv : n / 16 < n := Nat.div_lt_self (by omega) (by decide) + have hdigit : n % 16 < 16 := Nat.mod_lt _ (by decide) + rw [natToHexLower] + simp only [h16, ite_false] + rw [allBytes_append] + simp [allBytes, ih (n / 16) hdiv, (hexDigit_ok _ hdigit).1] + +theorem natToHex_no_break (n : Nat) : noBareLineBreak (natToHexLower n) := by + induction n using Nat.strongRecOn with + | ind n ih => + by_cases h16 : n < 16 + · have hmod : n % 16 = n := Nat.mod_eq_of_lt h16 + simp [natToHexLower, h16, hmod, noBareLineBreak] + exact ⟨(hexDigit_ok n h16).2.1, (hexDigit_ok n h16).2.2⟩ + · have hdiv : n / 16 < n := Nat.div_lt_self (by omega) (by decide) + have hdigit : n % 16 < 16 := Nat.mod_lt _ (by decide) + intro b hb + rw [natToHexLower] at hb + simp only [h16, ite_false, List.mem_append] at hb + cases hb with + | inl hb => exact ih _ hdiv b hb + | inr hb => + simp at hb + subst hb + exact ⟨(hexDigit_ok _ hdigit).2.1, (hexDigit_ok _ hdigit).2.2⟩ + +theorem takeWhile_of_all (p : UInt8 → Bool) (bs : List UInt8) (h : allBytes p bs = true) : + takeWhileBytes p bs = bs := by + induction bs with + | nil => simp [takeWhileBytes] + | cons b bs ih => + simp [allBytes] at h + rcases h with ⟨hb, hbs⟩ + simp [takeWhileBytes, hb, ih hbs] + +theorem parseChunkSize_hex (n : Nat) (h : n ≤ maxUInt64) : + parseChunkSize (natToHexLower n) = some n := by + have htake := takeWhile_of_all isHex (natToHexLower n) (allBytes_hex n) + have hdrop : dropBytes (natToHexLower n).length (natToHexLower n) = [] := by + simpa using dropBytes_append (natToHexLower n) [] + have hne := natToHexLower_ne_nil n + simp [parseChunkSize, htake, hdrop, parseHex_printHex n h, parseExt, hne] + +theorem decode_last (fuel : Nat) (acc : List UInt8) (h : 0 < fuel) : + decodeLoop fuel acc ([48] ++ crlf ++ crlf) = some (acc, [], []) := by + cases fuel with + | zero => omega + | succ fuel => + simp [decodeLoop, crlf, splitCRLF, parseChunkSize, parseHex, parseAcc, hexVal, parseExt, parseTrailers, + takeWhileBytes, dropBytes, isHex, isDigit] + +theorem decodeChunked_encodeChunked (body : List UInt8) (h : body.length ≤ maxUInt64) : + decodeChunked (encodeChunked body) = some (body, [], []) := by + cases body with + | nil => + simp [encodeChunked, decodeChunked, decodeLoop, crlf, splitCRLF, parseChunkSize, parseHex, parseAcc, hexVal, + parseExt, parseTrailers, takeWhileBytes, dropBytes, isHex, isDigit] + | cons b bs => + have hline := natToHex_no_break (b :: bs).length + have hhex := parseChunkSize_hex (b :: bs).length h + have hsplit := splitCRLF_of_line (natToHexLower (b :: bs).length) + ((b :: bs) ++ crlf ++ [48] ++ crlf ++ crlf) hline + have htake := takeBytes_append (b :: bs) (crlf ++ [48] ++ crlf ++ crlf) + have hdrop := dropBytes_append (b :: bs) (crlf ++ [48] ++ crlf ++ crlf) + simp [crlf, List.append_assoc, List.cons_append, List.length_cons] at hsplit htake hdrop hhex + simp [encodeChunked, decodeChunked, List.isEmpty, crlf, decodeLoop, hsplit] + rw [hhex] + simp [List.length_cons, List.length_append] + rw [hdrop, htake] + simp [crlf, splitCRLF, parseChunkSize, parseHex, parseAcc, hexVal, parseExt, parseTrailers, + takeWhileBytes, dropBytes, isHex, isDigit, decode_last] + +theorem tchar_ne (b : UInt8) (h : isTchar b = true) : b ≠ 13 ∧ b ≠ 10 ∧ b ≠ 32 ∧ b ≠ 58 := by + refine ⟨?_, ?_, ?_, ?_⟩ <;> intro hb <;> subst hb <;> simp [isTchar, isAlpha, isDigit] at h + +theorem vchar_ne (b : UInt8) (h : isVChar b = true) : b ≠ 13 ∧ b ≠ 10 ∧ b ≠ 32 := by + refine ⟨?_, ?_, ?_⟩ <;> intro hb <;> subst hb <;> simp [isVChar] at h + +theorem ne_beq_u8 {a b : UInt8} (h : a ≠ b) : (a == b) = false := by + cases hcmp : a == b + · rfl + · simp [beq_iff_eq] at hcmp + exact absurd hcmp h + +theorem noBare_all (p : UInt8 → Bool) (bs : List UInt8) + (hp : ∀ b, p b = true → b ≠ 13 ∧ b ≠ 10) (h : allBytes p bs = true) : + noBareLineBreak bs := by + induction bs with + | nil => simp [noBareLineBreak] + | cons b bs ih => + simp [allBytes] at h + rcases h with ⟨hb, hbs⟩ + intro x hx + simp at hx + rcases hx with rfl | hx + · exact hp x hb + · exact ih hbs x hx + +theorem splitByte_no (sep : UInt8) (bs : List UInt8) (h : ∀ b ∈ bs, (b == sep) = false) : + splitByte sep bs = [bs] := by + induction bs with + | nil => simp [splitByte] + | cons b bs ih => + have hb := h b (by simp) + have hbs : ∀ x ∈ bs, (x == sep) = false := fun x hx => h x (by simp [hx]) + simp [splitByte, ih hbs, hb] + +theorem splitByte_sep (sep : UInt8) (l r : List UInt8) (h : ∀ b ∈ l, (b == sep) = false) : + splitByte sep (l ++ sep :: r) = l :: splitByte sep r := by + induction l with + | nil => simp [splitByte] + | cons b l ih => + have hb := h b (by simp) + have hl : ∀ x ∈ l, (x == sep) = false := fun x hx => h x (by simp [hx]) + simp [splitByte, hb, ih hl] + +theorem version_no_sp (v : Version) : ∀ b ∈ versionBytes v, (b == 32) = false ∧ (b == 13) = false ∧ + (b == 10) = false := by + cases v <;> intro b hb <;> simp [versionBytes] at hb <;> + rcases hb with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl <;> decide + +theorem parseVersion_bytes (v : Version) : parseVersion (versionBytes v) = some v := by + cases v <;> simp [parseVersion, versionBytes, takeBytes, List.length, List.getLast?] + +theorem all_mem (p : UInt8 → Bool) (bs : List UInt8) (b : UInt8) + (h : allBytes p bs = true) (hb : b ∈ bs) : p b = true := by + induction bs with + | nil => simp at hb + | cons c cs ih => + simp [allBytes] at h + simp at hb + cases hb with + | inl hb => + subst hb + exact h.1 + | inr hb => + exact ih h.2 hb + +theorem bare_all (bs : List UInt8) (h : bareToken bs = true) : + allBytes isTchar bs = true ∧ bs ≠ [] := by + cases bs with + | nil => simp [bareToken] at h + | cons b bs => + simp [bareToken] at h + exact ⟨h, by simp⟩ + +theorem parseRequestLine_render (method target : List UInt8) (v : Version) + (hm : bareToken method = true) (ht : allBytes isVChar target = true) (hne : target ≠ []) : + parseRequestLine (method ++ [32] ++ target ++ [32] ++ versionBytes v) = + some (method, target, v) := by + have hmsp : ∀ b ∈ method, (b == 32) = false := fun b hb => + ne_beq_u8 (tchar_ne b (all_mem isTchar method b (bare_all method hm).1 hb)).2.2.1 + have htsp : ∀ b ∈ target, (b == 32) = false := fun b hb => + ne_beq_u8 (vchar_ne b (all_mem isVChar target b ht hb)).2.2 + have hvsp : ∀ b ∈ versionBytes v, (b == 32) = false := fun b hb => (version_no_sp v b hb).1 + have hmethod := splitByte_sep 32 method (target ++ 32 :: versionBytes v) hmsp + have htarget := splitByte_sep 32 target (versionBytes v) htsp + have hversion := splitByte_no 32 (versionBytes v) hvsp + unfold parseRequestLine + rw [show method ++ [32] ++ target ++ [32] ++ versionBytes v = + method ++ 32 :: (target ++ 32 :: versionBytes v) by simp [List.append_assoc, List.singleton_append]] + rw [hmethod, htarget, hversion] + simp [parseVersion_bytes, hm, hne, ht, vchars] + +theorem splitFirst_sep (sep : UInt8) (l r : List UInt8) (h : ∀ b ∈ l, (b == sep) = false) : + splitFirst sep (l ++ sep :: r) = some (l, r) := by + induction l with + | nil => simp [splitFirst] + | cons b l ih => + have hb := h b (by simp) + have hl : ∀ x ∈ l, (x == sep) = false := fun x hx => h x (by simp [hx]) + simp [splitFirst, hb, ih hl] + +theorem dropWhile_stop (b : UInt8) (bs : List UInt8) (h : isOWS b = false) : + dropWhileBytes isOWS (b :: bs) = b :: bs := by + simp [dropWhileBytes, h] + +theorem rev_cons (b : UInt8) (bs acc : List UInt8) : + revBytes (b :: bs) acc = revBytes bs (b :: acc) := by + simp [revBytes] + +theorem rev_append (bs acc : List UInt8) : + revBytes bs acc = revBytes bs [] ++ acc := by + induction bs generalizing acc with + | nil => simp [revBytes] + | cons b bs ih => + simp [revBytes, ih (b :: acc), ih [b], List.append_assoc] + +theorem rev_snoc (xs : List UInt8) (b : UInt8) : + revBytes (xs ++ [b]) [] = b :: revBytes xs [] := by + induction xs with + | nil => simp [revBytes] + | cons c xs ih => + rw [List.cons_append, revBytes, rev_append, ih, revBytes, rev_append, List.cons_append, + List.append_nil, ← rev_append] + +theorem rev_rev (bs : List UInt8) : revBytes (revBytes bs []) [] = bs := by + induction bs with + | nil => simp [revBytes] + | cons b bs ih => + rw [revBytes, rev_append bs [b], rev_snoc, ih] + +theorem not_ows (b : UInt8) (h : !isOWS b = true) : isOWS b = false := by + simpa using h + +theorem dropWhile_len (bs : List UInt8) : + (dropWhileBytes isOWS bs).length ≤ bs.length := by + induction bs with + | nil => simp [dropWhileBytes] + | cons b bs ih => + simp [dropWhileBytes] + split + · exact Nat.le_trans ih (Nat.le_succ _) + · simp + +theorem dropWhile_head (b : UInt8) (bs : List UInt8) + (h : dropWhileBytes isOWS (b :: bs) = b :: bs) : isOWS b = false := by + cases hO : isOWS b with + | false => rfl + | true => + simp [dropWhileBytes, hO] at h + have hlen := dropWhile_len bs + rw [h] at hlen + simp at hlen + omega + +theorem dropWhile_append_stop (xs ys : List UInt8) + (h : dropWhileBytes isOWS xs = xs) (hne : xs ≠ []) : + dropWhileBytes isOWS (xs ++ ys) = xs ++ ys := by + cases xs with + | nil => simp at hne + | cons b bs => + have hb := dropWhile_head b bs h + simp [dropWhileBytes, hb] + +theorem dropWhile_rev_noTrail (bs : List UInt8) (h : noTrail bs = true) : + dropWhileBytes isOWS (revBytes bs []) = revBytes bs [] := by + induction bs with + | nil => simp [revBytes, dropWhileBytes] + | cons b bs ih => + cases bs with + | nil => + simp [noTrail, revBytes, dropWhileBytes, not_ows b h] + | cons c cs => + simp [noTrail] at h + have ih' := ih h + have hne : revBytes (c :: cs) [] ≠ [] := by + rw [revBytes, rev_append] + intro hempty + simp [List.append_eq_nil_iff] at hempty + rw [revBytes, rev_append] + exact dropWhile_append_stop _ _ ih' hne + +theorem trim_canonical (v : List UInt8) (h : canonicalValue v = true) : trimOWS v = v := by + cases v with + | nil => simp [trimOWS, dropWhileBytes, revBytes, canonicalValue] + | cons b bs => + simp [canonicalValue] at h + have hb : isOWS b = false := h.1.1 + have htrail : noTrail (b :: bs) = true := h.1.2 + simp [trimOWS, dropWhile_stop b bs hb, dropWhile_rev_noTrail (b :: bs) htrail, rev_rev] + +theorem trim_space (v : List UInt8) (h : canonicalValue v = true) : trimOWS (32 :: v) = v := by + have h32 : isOWS 32 = true := by decide + cases v with + | nil => simp [trimOWS, dropWhileBytes, revBytes, h32] + | cons b bs => + simp [canonicalValue] at h + have hb : isOWS b = false := h.1.1 + have hfront : dropWhileBytes isOWS (32 :: b :: bs) = b :: bs := by + simp only [dropWhileBytes, h32, ite_true] + exact dropWhile_stop b bs hb + simp [trimOWS, hfront, dropWhile_rev_noTrail (b :: bs) h.1.2, rev_rev] + +theorem field_ne (b : UInt8) (h : isFieldByte b = true) : b ≠ 13 ∧ b ≠ 10 := by + refine ⟨?_, ?_⟩ <;> intro hb <;> subst hb <;> simp [isFieldByte, isVChar] at h + +theorem tchar_ne_htab (b : UInt8) (h : isTchar b = true) : b ≠ 9 := by + intro hb + subst hb + simp [isTchar, isAlpha, isDigit] at h + +theorem canonical_bytes (v : List UInt8) (h : canonicalValue v = true) : + allBytes isFieldByte v = true := by + cases v with + | nil => simp [allBytes] + | cons b bs => + simp [canonicalValue] at h + exact h.2 + +theorem noBare_append (xs ys : List UInt8) (hx : noBareLineBreak xs) (hy : noBareLineBreak ys) : + noBareLineBreak (xs ++ ys) := by + intro b hb + simp [List.mem_append] at hb + cases hb with + | inl hb => exact hx b hb + | inr hb => exact hy b hb + +theorem noBare_pair (x y : UInt8) (hx : x ≠ 13 ∧ x ≠ 10) (hy : y ≠ 13 ∧ y ≠ 10) : + noBareLineBreak [x, y] := by + intro b hb + simp at hb + rcases hb with rfl | rfl + · exact hx + · exact hy + +theorem field_line_noBare (name value : List UInt8) (hn : allBytes isTchar name = true) + (hv : allBytes isFieldByte value = true) : + noBareLineBreak (name ++ [58, 32] ++ value) := by + apply noBare_append + · apply noBare_append + · exact noBare_all isTchar name (fun b hb => + let h := tchar_ne b hb + ⟨h.1, h.2.1⟩) hn + · exact noBare_pair 58 32 (by decide) (by decide) + · exact noBare_all isFieldByte value field_ne hv + +theorem skip_cons (b : UInt8) (bs : List UInt8) (h : b ≠ 13) : skipEmpty (b :: bs) = b :: bs := by + unfold skipEmpty + split + · rename_i heq + simp at heq + exact absurd heq.1 h + · rfl + +theorem parseField_render (h : Header) (hc : canonicalHeader h = true) : + parseField (h.name ++ [58, 32] ++ h.value) = some h := by + have hn : bareToken h.name = true := by + simp [canonicalHeader] at hc + exact hc.1 + have hv : canonicalValue h.value = true := by + simp [canonicalHeader] at hc + exact hc.2 + have hall := (bare_all h.name hn).1 + have hne := (bare_all h.name hn).2 + have hsplit := splitFirst_sep 58 h.name (32 :: h.value) + (fun b hb => ne_beq_u8 (tchar_ne b (all_mem isTchar h.name b hall hb)).2.2.2) + have hbuf : h.name ++ [58, 32] ++ h.value = h.name ++ 58 :: (32 :: h.value) := by + simp [List.append_assoc, List.singleton_append] + cases hname : h.name with + | nil => simp [hname] at hne + | cons b bs => + have hb := all_mem isTchar h.name b hall (by simp [hname]) + have hsp : ((h.name).head? == some 32) = false := by + simp [hname, List.head?, ne_beq_u8 (tchar_ne b hb).2.2.1] + have htab : ((h.name).head? == some 9) = false := by + simp [hname, List.head?, ne_beq_u8 (tchar_ne_htab b hb)] + have hgoal : b :: bs ++ [58, 32] ++ h.value = h.name ++ 58 :: 32 :: h.value := by + simp [hname, List.append_assoc, List.cons_append, List.singleton_append] + simp [hname] at hsplit + simp [parseField, hname, List.cons_append, List.head?, + ne_beq_u8 (tchar_ne b hb).2.2.1, ne_beq_u8 (tchar_ne_htab b hb), + hsplit, trim_space h.value hv, hn, canonical_bytes h.value hv] + refine And.intro ?_ ?_ + · simpa [hname] using hn + · rw [← hname] + +theorem parseFields_render (hs : List Header) (rest : List UInt8) (fuel : Nat) + (hc : headersCanonical hs = true) (hf : hs.length + 1 ≤ fuel) : + parseFields fuel (renderHeaders hs ++ crlf ++ rest) = some (hs, rest) := by + induction hs generalizing fuel rest with + | nil => + cases fuel with + | zero => omega + | succ n => + simp [parseFields, renderHeaders, crlf, splitCRLF] + | cons h hs ih => + cases fuel with + | zero => simp at hf + | succ n => + have hc1 : canonicalHeader h = true := by + simp [headersCanonical] at hc + exact hc.1 + have hcs : headersCanonical hs = true := by + simp [headersCanonical] at hc + exact hc.2 + have hlen : hs.length + 1 ≤ n := by + simp at hf + omega + have hline := field_line_noBare h.name h.value (bare_all h.name + (by simp [canonicalHeader] at hc1; exact hc1.1)).1 (canonical_bytes h.value + (by simp [canonicalHeader] at hc1; exact hc1.2)) + have hsplit := splitCRLF_of_line (h.name ++ [58, 32] ++ h.value) + (renderHeaders hs ++ crlf ++ rest) hline + have hfield := parseField_render h hc1 + have hsplit' : + splitCRLF (h.name ++ 58 :: 32 :: (h.value ++ 13 :: 10 :: (renderHeaders hs ++ 13 :: 10 :: rest))) = + some (h.name ++ [58, 32] ++ h.value, renderHeaders hs ++ crlf ++ rest) := by + simpa [crlf, List.append_assoc, List.singleton_append, List.cons_append] using hsplit + simp only [crlf, List.append_assoc] at hsplit + have hfield' : parseField (h.name ++ ([58, 32] ++ h.value)) = some h := by + simpa [List.append_assoc] using hfield + simp only [parseFields, renderHeaders, renderHeader, crlf, List.append_assoc, hsplit] + split + · rename_i heq + injection heq + · rename_i rest' heq + injection heq with hnil + simp [List.append_eq_nil_iff] at hnil + · rename_i line rest' _ heq + injection heq with hpair + injection hpair with hline hrest + subst hline + subst hrest + rw [hfield'] + split + · rename_i heq + injection heq + · rename_i h' heq + injection heq with hh + subst hh + have hbuf : renderHeaders hs ++ ([13, 10] ++ rest) = renderHeaders hs ++ crlf ++ rest := by + simp [crlf, List.append_assoc] + rw [hbuf, ih rest n hcs hlen] + +theorem render_len (hs : List Header) : hs.length ≤ (renderHeaders hs).length := by + induction hs with + | nil => simp [renderHeaders] + | cons h hs ih => + simp [renderHeaders, renderHeader, List.length_append] + omega + +theorem parseHead_render (hd : RequestHead) (rest : List UInt8) (ha : acceptHead hd = true) : + parseHead (renderHead hd ++ rest) = some (hd, rest) := by + simp [acceptHead, Bool.and_eq_true, List.isEmpty] at ha + have hm : bareToken hd.method = true := ha.1.1.1.1 + have htne : hd.target ≠ [] := by + intro hnil + have ha' := ha + simp [hnil] at ha' + have ht : allBytes isVChar hd.target = true := by + have hvchars : vchars hd.target = true := ha.1.1.2 + simpa [vchars] using hvchars + have hhc : headersCanonical hd.headers = true := ha.1.2 + have hok : requestOK hd.method hd.target hd.version hd.headers = true := ha.2 + have hline := parseRequestLine_render hd.method hd.target hd.version hm ht htne + have hbare : noBareLineBreak (hd.method ++ [32] ++ hd.target ++ [32] ++ versionBytes hd.version) := by + intro b hb + simp [List.mem_append] at hb + rcases hb with hb | rfl | hb | rfl | hb + · exact ⟨(tchar_ne b (all_mem isTchar hd.method b (bare_all hd.method hm).1 hb)).1, + (tchar_ne b (all_mem isTchar hd.method b (bare_all hd.method hm).1 hb)).2.1⟩ + · decide + · exact ⟨(vchar_ne b (all_mem isVChar hd.target b ht hb)).1, + (vchar_ne b (all_mem isVChar hd.target b ht hb)).2.1⟩ + · decide + · have hv := version_no_sp hd.version b hb + refine ⟨?_, ?_⟩ <;> intro heq <;> subst heq <;> simp at hv + have hsplit := splitCRLF_of_line + (hd.method ++ [32] ++ hd.target ++ [32] ++ versionBytes hd.version) + (renderHeaders hd.headers ++ crlf ++ rest) hbare + have hfuel : hd.headers.length + 1 ≤ (renderHeaders hd.headers ++ crlf ++ rest).length := by + have hlen := render_len hd.headers + simp [List.length_append, crlf] + omega + have hfields := parseFields_render hd.headers rest _ hhc hfuel + have hskip : skipEmpty (renderHead hd ++ rest) = renderHead hd ++ rest := by + cases hmethod : hd.method with + | nil => simp [hmethod, bareToken] at hm + | cons b bs => + have hb : b ≠ 13 := + (tchar_ne b (all_mem isTchar hd.method b (bare_all hd.method hm).1 (by simp [hmethod]))).1 + simp [renderHead, hmethod, List.cons_append] + exact skip_cons b _ hb + have hskip2 : + skipEmpty (hd.method ++ 32 :: (hd.target ++ 32 :: (versionBytes hd.version ++ 13 :: 10 :: + (renderHeaders hd.headers ++ 13 :: 10 :: rest)))) = + hd.method ++ 32 :: (hd.target ++ 32 :: (versionBytes hd.version ++ 13 :: 10 :: + (renderHeaders hd.headers ++ 13 :: 10 :: rest))) := by + cases hmethod : hd.method with + | nil => simp [hmethod, bareToken] at hm + | cons b bs => + have hb : b ≠ 13 := + (tchar_ne b (all_mem isTchar hd.method b (bare_all hd.method hm).1 (by simp [hmethod]))).1 + simp [hmethod, List.cons_append] + exact skip_cons b _ hb + have hsplit2 : + splitCRLF (hd.method ++ 32 :: (hd.target ++ 32 :: (versionBytes hd.version ++ 13 :: 10 :: + (renderHeaders hd.headers ++ 13 :: 10 :: rest)))) = + some (hd.method ++ [32] ++ hd.target ++ [32] ++ versionBytes hd.version, + renderHeaders hd.headers ++ crlf ++ rest) := by + simpa [crlf, List.append_assoc, List.singleton_append, List.cons_append] using hsplit + simp only [renderHead, crlf, List.append_assoc] at hskip hsplit + simp only [parseHead, renderHead, crlf, List.append_assoc] + rw [hskip, hsplit] + split + · rename_i heq + injection heq + · rename_i _ l rest' heq + simp only [List.append_assoc] at hline + injection heq with hpair + injection hpair with hline' hrest + subst hline' + subst hrest + rw [hline] + split + · rename_i heq + injection heq + · rename_i parsed heq + injection heq with hparsed + cases hparsed + simp only [List.append_assoc, crlf] at hfields + rw [hfields] + split + · rename_i heq + injection heq + · rename_i headers rest' heq + injection heq with hpair + injection hpair with hh hr + subst hh + subst hr + simp [hok] + +theorem takeBytes_length (bs : List UInt8) : takeBytes bs.length bs = bs := by + induction bs with + | nil => simp [takeBytes] + | cons b bs ih => simp [takeBytes, List.length, ih] + +theorem dropBytes_length (bs : List UInt8) : dropBytes bs.length bs = [] := by + induction bs with + | nil => simp [dropBytes] + | cons _ bs ih => simp [dropBytes, List.length, ih] + +theorem parseRequest_serializeFixed (hd : RequestHead) (body wire : List UInt8) + (hs : serializeFixed hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = hd.version ∧ + r.body = body ∧ r.rest = [] := by + simp [serializeFixed] at hs + split at hs + · next n hf => + split at hs + · next hif => + cases hs + rcases hif with ⟨hlen, ha⟩ + have hhead := parseHead_render hd body ha + refine ⟨{ head := hd, body := body, rest := [] }, ?_, rfl, rfl, rfl, rfl, rfl⟩ + simp [parseRequest, hhead, hf, hlen, takeBytes_length, dropBytes_length] + · cases hs + · cases hs + +theorem parseRequest_serializeChunked (hd : RequestHead) (body wire : List UInt8) + (hs : serializeChunked hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = .http11 ∧ + r.body = body ∧ r.rest = [] := by + simp [serializeChunked, Bool.and_eq_true] at hs + rcases hs with ⟨⟨⟨⟨hv, hf⟩, ha⟩, hlen⟩, hwire⟩ + subst hwire + have hhead := parseHead_render hd (encodeChunked body) ha + have hdec := decodeChunked_encodeChunked body hlen + refine ⟨{ head := hd, body := body, rest := [] }, ?_, rfl, rfl, ?_, rfl, rfl⟩ + · simp [parseRequest, hhead, hf, hdec] + · simpa using hv + +theorem takeBytes_le (n : Nat) (bs extra : List UInt8) (h : n ≤ bs.length) : + takeBytes n (bs ++ extra) = takeBytes n bs := by + induction n generalizing bs with + | zero => simp [takeBytes] + | succ n ih => + cases bs with + | nil => simp at h + | cons b bs => + simp [List.length] at h + simp [takeBytes, List.cons_append] + exact ih bs h + +theorem dropBytes_le (n : Nat) (bs extra : List UInt8) (h : n ≤ bs.length) : + dropBytes n (bs ++ extra) = dropBytes n bs ++ extra := by + induction n generalizing bs with + | zero => simp [dropBytes] + | succ n ih => + cases bs with + | nil => simp at h + | cons _ bs => + simp [dropBytes, List.length] at h ⊢ + exact ih bs h + +theorem parseFields_prefix (fuel fuel' : Nat) (bs : List UInt8) (hs : List Header) + (rest extra : List UInt8) + (h : parseFields fuel bs = some (hs, rest)) (hf : fuel ≤ fuel') : + parseFields fuel' (bs ++ extra) = some (hs, rest ++ extra) := by + induction fuel generalizing bs hs rest fuel' with + | zero => simp [parseFields] at h + | succ fuel ih => + cases fuel' with + | zero => omega + | succ fuel' => + simp [parseFields] at h + cases hsp : splitCRLF bs with + | none => simp [hsp] at h + | some pair => + simp [hsp] at h + rcases pair with ⟨line, rest0⟩ + have hpre := splitCRLF_prefix_free bs line rest0 extra hsp + cases line with + | nil => + simp at h + rcases h with ⟨rfl, rfl⟩ + simp [parseFields, hpre] + | cons b line => + simp at h + cases hfield : parseField (b :: line) with + | none => simp [hfield] at h + | some hdr => + simp [hfield] at h + cases hrest : parseFields fuel rest0 with + | none => simp [hrest] at h + | some parsed => + simp [hrest] at h + rcases parsed with ⟨hs0, rest'⟩ + rcases h with ⟨rfl, rfl, rfl⟩ + have ih' := ih fuel' rest0 hs0 rest' hrest (by omega) + simp [parseFields, hpre, hfield, ih'] + +theorem skip_crlf (bs : List UInt8) : skipEmpty (13 :: 10 :: bs) = skipEmpty bs := by + simp [skipEmpty] + +theorem skip_not (b c : UInt8) (bs : List UInt8) (h : b ≠ 13 ∨ c ≠ 10) : + skipEmpty (b :: c :: bs) = b :: c :: bs := by + unfold skipEmpty + split + · rename_i heq + simp at heq + rcases h with h | h + · exact absurd heq.1 h + · exact absurd heq.2.1 h + · rfl + +theorem skip_one (b : UInt8) : skipEmpty [b] = [b] := by + unfold skipEmpty + split + · rename_i heq + simp at heq + · rfl + +theorem skipEmpty_append (bs extra : List UInt8) (b : UInt8) (rest : List UInt8) + (h : skipEmpty bs = b :: rest) (hb : b ≠ 13) : + skipEmpty (bs ++ extra) = (b :: rest) ++ extra := by + match bs with + | [] => simp [skipEmpty] at h + | [x] => + simp [skip_one] at h + rcases h with ⟨rfl, rfl⟩ + simp [List.singleton_append, skip_cons x extra hb] + | 13 :: 10 :: ys => + simp [skip_crlf] at h + exact skipEmpty_append ys extra b rest h hb + | x :: y :: ys => + by_cases hx : x = 13 + · subst hx + by_cases hy : y = 10 + · subst hy + simp [skip_crlf] at h + exact skipEmpty_append ys extra b rest h hb + · have hid := skip_not 13 y (ys ++ extra) (Or.inr hy) + simp [skip_not 13 y ys (Or.inr hy), List.cons_append] at h + rcases h with ⟨rfl, rfl⟩ + simp [hid, List.cons_append] + · have hid := skip_not x y (ys ++ extra) (Or.inl hx) + simp [skip_not x y ys (Or.inl hx), List.cons_append] at h + rcases h with ⟨rfl, rfl⟩ + simp [hid, List.cons_append] +termination_by bs.length + +theorem method_head (line : List UInt8) (method target : List UInt8) (v : Version) + (h : parseRequestLine line = some (method, target, v)) : + ∃ b rest, line = b :: rest ∧ b ≠ 13 := by + cases line with + | nil => simp [parseRequestLine, splitByte] at h + | cons c cs => + refine ⟨c, cs, rfl, ?_⟩ + intro hc + subst hc + cases hcs : splitByte 32 cs with + | nil => simp [parseRequestLine, splitByte, hcs, bareToken, allBytes, isTchar, isAlpha, isDigit] at h + | cons p ps => + cases ps with + | nil => simp [parseRequestLine, splitByte, hcs] at h + | cons q qs => + cases qs with + | nil => simp [parseRequestLine, splitByte, hcs] at h + | cons r rs => + cases rs with + | nil => + simp [parseRequestLine, splitByte, hcs, bareToken, allBytes, isTchar, isAlpha, isDigit] at h + split at h <;> simp at h + | cons => simp [parseRequestLine, splitByte, hcs] at h + +theorem parseHead_prefix (bs extra : List UInt8) (hd : RequestHead) (rest : List UInt8) + (h : parseHead bs = some (hd, rest)) : + parseHead (bs ++ extra) = some (hd, rest ++ extra) := by + unfold parseHead at h + cases hsk : skipEmpty bs with + | nil => simp [hsk, splitCRLF] at h + | cons b skipped => + cases hsp : splitCRLF (b :: skipped) with + | none => simp [hsk, hsp] at h + | some got => + obtain ⟨line, after⟩ := got + cases hreq : parseRequestLine line with + | none => simp [hsk, hsp, hreq] at h + | some parsed => + obtain ⟨method, target, v⟩ := parsed + obtain ⟨c, _, hline, hc⟩ := method_head line method target v hreq + cases hf : parseFields after.length after with + | none => simp [hsk, hsp, hreq, hf] at h + | some fields => + obtain ⟨headers, rest'⟩ := fields + simp [hsk, hsp, hreq, hf] at h + rcases h with ⟨hok, rfl, rfl⟩ + have hbuf := (splitCRLF_sound (b :: skipped) line after).mp hsp + have hb : b ≠ 13 := by + rcases hbuf with ⟨_, hbuf⟩ + simp [hline, List.cons_append] at hbuf + rcases hbuf with ⟨hb', _⟩ + exact hb' ▸ hc + have happ := skipEmpty_append bs extra b skipped hsk hb + have hpre := splitCRLF_prefix_free (b :: skipped) line after extra hsp + have hmore : after.length ≤ (after ++ extra).length := by simp [List.length_append] + have hfields := parseFields_prefix after.length (after ++ extra).length after headers rest' + extra hf hmore + simp [List.cons_append] at hpre happ + simp [parseHead, happ, hpre, hreq] + have hfields' : parseFields (after.length + extra.length) (after ++ extra) = + some (headers, rest' ++ extra) := by + simpa [List.length_append] using hfields + rw [hfields'] + simp [hok] + +theorem parseTrailers_prefix (fuel fuel' : Nat) (bs : List UInt8) (hs : List Header) + (rest extra : List UInt8) + (h : parseTrailers fuel bs = some (hs, rest)) (hf : fuel ≤ fuel') : + parseTrailers fuel' (bs ++ extra) = some (hs, rest ++ extra) := by + induction fuel generalizing bs hs rest fuel' with + | zero => simp [parseTrailers] at h + | succ fuel ih => + cases fuel' with + | zero => omega + | succ fuel' => + simp [parseTrailers] at h + cases hsp : splitCRLF bs with + | none => simp [hsp] at h + | some pair => + simp [hsp] at h + rcases pair with ⟨line, rest0⟩ + have hpre := splitCRLF_prefix_free bs line rest0 extra hsp + cases line with + | nil => + simp at h + rcases h with ⟨rfl, rfl⟩ + simp [parseTrailers, hpre] + | cons b line => + simp at h + cases hfield : parseField (b :: line) with + | none => simp [hfield] at h + | some hdr => + simp [hfield] at h + cases hrest : parseTrailers fuel rest0 with + | none => simp [hrest] at h + | some parsed => + simp [hrest] at h + rcases parsed with ⟨hs0, rest'⟩ + rcases h with ⟨rfl, rfl, rfl⟩ + have ih' := ih fuel' rest0 hs0 rest' hrest (by omega) + simp [parseTrailers, hpre, hfield, ih'] + +theorem decodeLoop_prefix (fuel fuel' : Nat) (acc buf body : List UInt8) (ts : List Header) + (rest extra : List UInt8) + (h : decodeLoop fuel acc buf = some (body, ts, rest)) (hf : fuel ≤ fuel') : + decodeLoop fuel' acc (buf ++ extra) = some (body, ts, rest ++ extra) := by + induction fuel generalizing acc buf body ts rest fuel' with + | zero => simp [decodeLoop] at h + | succ fuel ih => + cases fuel' with + | zero => omega + | succ fuel' => + simp [decodeLoop] at h + cases hsp : splitCRLF buf with + | none => simp [hsp] at h + | some got => + obtain ⟨line, after⟩ := got + have hpre := splitCRLF_prefix_free buf line after extra hsp + simp [hsp] at h + cases hsz : parseChunkSize line with + | none => simp [hsz] at h + | some n => + by_cases hz : n = 0 + · subst hz + simp [hsz] at h + cases htr : parseTrailers after.length after with + | none => simp [htr] at h + | some parsed => + simp [htr] at h + rcases parsed with ⟨ts0, rest0⟩ + rcases h with ⟨rfl, rfl, rfl⟩ + have hmore : after.length ≤ (after ++ extra).length := by simp [List.length_append] + have htr' := parseTrailers_prefix after.length (after ++ extra).length after ts0 rest0 + extra htr hmore + have htr'' : parseTrailers (after.length + extra.length) (after ++ extra) = + some (ts0, rest0 ++ extra) := by + simpa [List.length_append] using htr' + simp [decodeLoop, hpre, hsz, htr''] + · simp [hsz, hz] at h + split at h + · rename_i _ rest0 heq + rcases h with ⟨hlen, hrec⟩ + have htake := takeBytes_le n after extra (by omega) + have hdrop := dropBytes_le n after extra (by omega) + simp [decodeLoop, hpre, hsz, htake, hdrop, heq] + refine And.intro ?_ (ih fuel' (acc ++ takeBytes n after) rest0 body ts rest hrec (by omega)) + · omega + · simp at h + +theorem decodeChunked_prefix_free (buf body rest extra : List UInt8) (trailers : List Header) + (h : decodeChunked buf = some (body, trailers, rest)) : + decodeChunked (buf ++ extra) = some (body, trailers, rest ++ extra) := by + simp [decodeChunked] at h ⊢ + exact decodeLoop_prefix (buf.length + 1) (buf.length + extra.length + 1) [] buf body trailers + rest extra h (by omega) + +theorem parseRequest_prefix_free (stream extra : List UInt8) (r : FramedRequest) + (h : parseRequest stream = some r) : + parseRequest (stream ++ extra) = some { r with rest := r.rest ++ extra } := by + simp [parseRequest] at h + cases hh : parseHead stream with + | none => simp [hh] at h + | some got => + obtain ⟨hd, rest⟩ := got + have hp := parseHead_prefix stream extra hd rest hh + cases hf : determineFraming hd.version hd.headers with + | none => simp [hh, hf] at h + | some fr => + cases fr with + | fixed n => + simp [hh, hf] at h + rcases h with ⟨hlen, rfl⟩ + have htake := takeBytes_le n rest extra hlen + have hdrop := dropBytes_le n rest extra hlen + have hlong : ¬ (rest ++ extra).length < n := by + simp [List.length_append] + omega + simp [parseRequest, hp, hf, htake, hdrop] + omega + | chunked => + simp [hh, hf] at h + cases hdec : decodeChunked rest with + | none => simp [hdec] at h + | some parsed => + simp [hdec] at h + obtain ⟨body, ts, rest'⟩ := parsed + cases h + have hdec' := decodeChunked_prefix_free rest body rest' extra ts hdec + simp [parseRequest, hp, hf, hdec'] + +theorem parseRequest_stream_unique (stream tail1 tail2 : List UInt8) + (msgs1 msgs2 : List (RequestHead × List UInt8)) + (hlen : msgs1.length = msgs2.length) + (h1 : Spec.StreamFrames msgs1 tail1 stream) + (h2 : Spec.StreamFrames msgs2 tail2 stream) : + msgs1 = msgs2 ∧ tail1 = tail2 := by + induction h1 generalizing msgs2 tail2 with + | nil => + cases h2 with + | nil => exact ⟨rfl, rfl⟩ + | cons _ _ => simp at hlen + | cons hp hrest ih => + cases h2 with + | nil => simp at hlen + | cons hp2 hrest2 => + have hr : _ = _ := hp.symm.trans hp2 + cases hr + simp at hlen + have ih' := ih _ _ hlen hrest2 + rcases ih' with ⟨rfl, rfl⟩ + exact ⟨rfl, rfl⟩ + +theorem reject_smuggling_vectors : + ∀ v, v ∈ Spec.smugglingVectors → parseRequest v = none := by + intro v hv + simp [Spec.smugglingVectors] at hv + rcases hv with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl + all_goals decide + +end Http diff --git a/Http/Parse.lean b/Http/Parse.lean new file mode 100644 index 0000000..c50a3dd --- /dev/null +++ b/Http/Parse.lean @@ -0,0 +1,391 @@ +import Http.Types +import Http.Bytes +import Http.Framing + +namespace Http + +def crlf : List UInt8 := [13, 10] + +def isFieldByte (b : UInt8) : Bool := + b == 9 || b == 32 || isVChar b || b ≥ 128 + +def skipEmpty : List UInt8 → List UInt8 + | 13 :: 10 :: rest => skipEmpty rest + | bs => bs + +/-- Split on the first `sep`. -/ +def splitFirst (sep : UInt8) : List UInt8 → Option (List UInt8 × List UInt8) + | [] => none + | b :: bs => + if b == sep then some ([], bs) + else + match splitFirst sep bs with + | none => none + | some (l, r) => some (b :: l, r) + +def parseVersion (bs : List UInt8) : Option Version := + if bs.length = 8 && takeBytes 7 bs == [72, 84, 84, 80, 47, 49, 46] then + match bs.getLast? with + | some d => + if d == 48 then some .http10 + else if 49 ≤ d && d ≤ 57 then some .http11 + else none + | none => none + else + none + +def versionBytes : Version → List UInt8 + | .http10 => [72, 84, 84, 80, 47, 49, 46, 48] + | .http11 => [72, 84, 84, 80, 47, 49, 46, 49] + +def startsWith (bs pre : List UInt8) : Bool := + takeBytes pre.length bs == pre + +def startsWithLower (bs pre : List UInt8) : Bool := + lowerBytes (takeBytes pre.length bs) == lowerBytes pre && pre.length ≤ bs.length + +def httpPrefix : List UInt8 := [104, 116, 116, 112, 58] +def httpsPrefix : List UInt8 := [104, 116, 116, 112, 115, 58] + +def vchars (bs : List UInt8) : Bool := + allBytes isVChar bs + +/-- host [ ":" port ], host nonempty, port absent or digits (empty port allowed). -/ +def hostPort (bs : List UInt8) : Bool := + if bs.isEmpty || !vchars bs then + false + else if bs.head? == some 91 then + match splitFirst 93 bs with + | none => false + | some (lit, rest) => + if lit.length ≤ 1 then false + else + match rest with + | [] => true + | 58 :: port => port.isEmpty || digits port + | _ => false + else + match splitFirst 58 bs with + | none => true + | some (host, port) => !host.isEmpty && (port.isEmpty || digits port) + +def noUserinfo (bs : List UInt8) : Bool := + !(anyBytes (· == 64) bs) + +/-- `( "http" / "https" ) "://" host [ ":" port ] path-abempty [ "?" query ]` with a nonempty host and no userinfo. -/ +def httpTarget (bs : List UInt8) : Bool := + let pre := if startsWithLower bs httpsPrefix then httpsPrefix else httpPrefix + if !startsWithLower bs pre then + false + else + let rest := dropBytes pre.length bs + if !startsWith rest ([47, 47]) then + false + else + let authAnd := dropBytes 2 rest + let auth := takeWhileBytes (fun b => b != 47 && b != 63) authAnd + let path := dropWhileBytes (fun b => b != 47 && b != 63) authAnd + vchars bs && noUserinfo auth && hostPort auth && vchars path && !(anyBytes (· == 35) path) + +def originTarget (bs : List UInt8) : Bool := + bs.head? == some 47 && vchars bs && !(anyBytes (· == 35) bs) + +def authorityTarget (bs : List UInt8) : Bool := + hostPort bs && !(anyBytes (· == 47) bs) && !(anyBytes (· == 63) bs) && !(anyBytes (· == 35) bs) + +def schemeRest (bs : List UInt8) : Bool := + match splitFirst 58 bs with + | none => false + | some (scheme, rest) => + match scheme with + | [] => false + | s :: ss => + isAlpha s && allBytes (fun b => isAlpha b || isDigit b || b == 43 || b == 45 || b == 46) ss && + vchars rest && !(anyBytes (· == 35) rest) && + !(lowerBytes scheme == [104, 116, 116, 112]) && + !(lowerBytes scheme == [104, 116, 116, 112, 115]) + +def connectPort (port : List UInt8) : Bool := + match parseDec port with + | some n => digits port && 1 ≤ n && n ≤ 65535 + | none => false + +/-- CONNECT target: nonempty host, colon, port 1..65535. Leading zeros allowed. -/ +def connectTarget (bs : List UInt8) : Bool := + if bs.head? == some 91 then + match splitFirst 93 bs with + | none => false + | some (_, rest) => + match rest with + | 58 :: port => connectPort port && vchars bs && !(anyBytes (· == 64) bs) + | _ => false + else + match splitFirst 58 bs with + | some (host, port) => !host.isEmpty && connectPort port && vchars bs && !(anyBytes (· == 64) bs) && + !(anyBytes (· == 58) host) + | none => false + +def targetOK (method target : List UInt8) : Bool := + if method == connectName then + connectTarget target + else if startsWithLower target httpPrefix || startsWithLower target httpsPrefix then + httpTarget target + else if target == [42] then + true + else if target.head? == some 47 then + originTarget target + else if schemeRest target then + true + else + authorityTarget target + +def hostValueOK (v : List UInt8) : Bool := + v.isEmpty || (hostPort v && !(v.head? == some 58)) + +def countName (n : List UInt8) : List Header → Nat + | [] => 0 + | h :: hs => (if lowerBytes h.name == n then 1 else 0) + countName n hs + +def hostsOK : List Header → Bool + | [] => true + | h :: hs => + (if lowerBytes h.name == hostNameLower then hostValueOK h.value else true) && hostsOK hs + +def requestOK (method target : List UInt8) (v : Version) (hdrs : List Header) : Bool := + let hosts := countName hostNameLower hdrs + let hostVal := hostsOK hdrs + let hostCount := if v = .http11 then hosts == 1 else hosts ≤ 1 + let connectContent := + if method == connectName then + (teValues hdrs).isEmpty && (match clState hdrs with | .absent => true | .ok 0 => true | _ => false) + else + true + targetOK method target && hostCount && hostVal && connectContent && + !(target.head? == some 58) + +def parseField (line : List UInt8) : Option Header := + if line.head? == some 32 || line.head? == some 9 then + none + else + match splitFirst 58 line with + | none => none + | some (name, raw) => + let value := trimOWS raw + if bareToken name && allBytes isFieldByte value then + some { name, value } + else + none + +def parseFields (fuel : Nat) (bs : List UInt8) : Option (List Header × List UInt8) := + match fuel with + | 0 => none + | fuel + 1 => + match splitCRLF bs with + | none => none + | some ([], rest) => some ([], rest) + | some (line, rest) => + match parseField line with + | none => none + | some h => + match parseFields fuel rest with + | none => none + | some (hs, rest') => some (h :: hs, rest') + +def parseRequestLine (line : List UInt8) : Option (List UInt8 × List UInt8 × Version) := + match splitByte 32 line with + | [method, target, version] => + match parseVersion version with + | none => none + | some v => + if bareToken method && !target.isEmpty && vchars target then + some (method, target, v) + else + none + | _ => none + +def parseHead (bs : List UInt8) : Option (RequestHead × List UInt8) := + match splitCRLF (skipEmpty bs) with + | none => none + | some (line, rest) => + match parseRequestLine line with + | none => none + | some (method, target, v) => + match parseFields rest.length rest with + | none => none + | some (headers, rest') => + if requestOK method target v headers then + some ({ method, target, version := v, headers }, rest') + else + none + +-- Chunk extensions: token name and an optional token or quoted-string. + +def skipOWS : List UInt8 → List UInt8 + | 32 :: bs => skipOWS bs + | 9 :: bs => skipOWS bs + | bs => bs + +def takeToken (bs : List UInt8) : Option (List UInt8 × List UInt8) := + let tok := takeWhileBytes isTchar bs + if tok.isEmpty then none else some (tok, dropBytes tok.length bs) + +def isQdtext (b : UInt8) : Bool := + b == 9 || b == 32 || b == 33 || (35 ≤ b && b ≤ 91) || (93 ≤ b && b ≤ 126) || b ≥ 128 + +def isQuotedChar (b : UInt8) : Bool := + b == 9 || b == 32 || isVChar b || b ≥ 128 + +def parseQuoted : List UInt8 → Option (List UInt8) + | [] => none + | 34 :: rest => some rest + | 92 :: c :: rest => if isQuotedChar c then parseQuoted rest else none + | b :: rest => if isQdtext b then parseQuoted rest else none + +/-- One extension, including the leading semicolon. Returns the unread suffix. -/ +def parseOneExt (bs : List UInt8) : Option (List UInt8) := + match skipOWS bs with + | 59 :: rest => + match takeToken (skipOWS rest) with + | none => none + | some (_, after) => + match skipOWS after with + | 61 :: restEq => + match skipOWS restEq with + | 34 :: quoted => parseQuoted quoted + | other => + match takeToken other with + | some (_, r) => some r + | none => none + | other => some other + | _ => none + +def parseExt (fuel : Nat) : List UInt8 → Bool + | [] => true + | bs => + match fuel with + | 0 => false + | fuel + 1 => + match parseOneExt bs with + | none => false + | some rest => + if rest.length < bs.length then parseExt fuel rest else false + +def parseChunkSize (line : List UInt8) : Option Nat := + let digits := takeWhileBytes isHex line + if digits.isEmpty then + none + else + match parseHex digits with + | none => none + | some n => + if parseExt line.length (dropBytes digits.length line) then some n else none + +def parseTrailers (fuel : Nat) (bs : List UInt8) : Option (List Header × List UInt8) := + match fuel with + | 0 => none + | fuel + 1 => + match splitCRLF bs with + | none => none + | some ([], rest) => some ([], rest) + | some (line, rest) => + match parseField line with + | none => none + | some h => + match parseTrailers fuel rest with + | none => none + | some (hs, rest') => some (h :: hs, rest') + +def decodeLoop (fuel : Nat) (acc : List UInt8) (buf : List UInt8) : + Option (List UInt8 × List Header × List UInt8) := + match fuel with + | 0 => none + | fuel + 1 => + match splitCRLF buf with + | none => none + | some (line, rest) => + match parseChunkSize line with + | none => none + | some 0 => + match parseTrailers rest.length rest with + | none => none + | some (ts, rest') => some (acc, ts, rest') + | some n => + if rest.length < n + 2 then + none + else + let data := takeBytes n rest + match dropBytes n rest with + | 13 :: 10 :: rest' => decodeLoop fuel (acc ++ data) rest' + | _ => none + +def decodeChunked (buf : List UInt8) : Option (List UInt8 × List Header × List UInt8) := + decodeLoop (buf.length + 1) [] buf + +def encodeChunked (body : List UInt8) : List UInt8 := + if body.isEmpty then + [48] ++ crlf ++ crlf + else + natToHexLower body.length ++ crlf ++ body ++ crlf ++ [48] ++ crlf ++ crlf + +def renderHeader (h : Header) : List UInt8 := + h.name ++ [58, 32] ++ h.value ++ crlf + +def renderHeaders : List Header → List UInt8 + | [] => [] + | h :: hs => renderHeader h ++ renderHeaders hs + +def renderHead (hd : RequestHead) : List UInt8 := + hd.method ++ [32] ++ hd.target ++ [32] ++ versionBytes hd.version ++ crlf ++ + renderHeaders hd.headers ++ crlf + +def noTrail : List UInt8 → Bool + | [] => true + | [b] => !isOWS b + | _ :: bs => noTrail bs + +/-- No leading or trailing optional whitespace, and only field bytes. -/ +def canonicalValue (v : List UInt8) : Bool := + match v with + | [] => true + | b :: bs => !isOWS b && noTrail (b :: bs) && allBytes isFieldByte v + +def canonicalHeader (h : Header) : Bool := + bareToken h.name && canonicalValue h.value + +def headersCanonical : List Header → Bool + | [] => true + | h :: hs => canonicalHeader h && headersCanonical hs + +def acceptHead (hd : RequestHead) : Bool := + bareToken hd.method && !hd.target.isEmpty && vchars hd.target && + headersCanonical hd.headers && + requestOK hd.method hd.target hd.version hd.headers + +def serializeFixed (hd : RequestHead) (body : List UInt8) : Option (List UInt8) := + match determineFraming hd.version hd.headers with + | some (.fixed n) => + if n = body.length && acceptHead hd then some (renderHead hd ++ body) else none + | _ => none + +def serializeChunked (hd : RequestHead) (body : List UInt8) : Option (List UInt8) := + if hd.version = .http11 && determineFraming hd.version hd.headers = some .chunked && + acceptHead hd && body.length ≤ maxUInt64 then + some (renderHead hd ++ encodeChunked body) + else + none + +def parseRequest (bs : List UInt8) : Option FramedRequest := + match parseHead bs with + | none => none + | some (hd, rest) => + match determineFraming hd.version hd.headers with + | none => none + | some (.fixed n) => + if rest.length < n then none + else some { head := hd, body := takeBytes n rest, rest := dropBytes n rest } + | some .chunked => + match decodeChunked rest with + | none => none + | some (body, _, rest') => some { head := hd, body, rest := rest' } + +end Http diff --git a/Http/Proofs.lean b/Http/Proofs.lean new file mode 100644 index 0000000..4938341 --- /dev/null +++ b/Http/Proofs.lean @@ -0,0 +1,103 @@ +import Http.Lemmas + +/-! +The proofs of the statements in `Http/Spec.lean`, under the same names. +This file does not import the challenge module. +-/ + +namespace Http.Spec + +open Http + +theorem splitCRLF_sound (bs line rest : List UInt8) : + splitCRLF bs = some (line, rest) ↔ + NoBareLineBreak line ∧ bs = line ++ [0x0D, 0x0A] ++ rest := by + simpa [NoBareLineBreak, noBareLineBreak] using Http.splitCRLF_sound bs line rest + +theorem splitCRLF_prefix_free (bs line rest extra : List UInt8) + (h : splitCRLF bs = some (line, rest)) : + splitCRLF (bs ++ extra) = some (line, rest ++ extra) := + Http.splitCRLF_prefix_free bs line rest extra h + +theorem parseDec_printDec (n : Nat) (h : n ≤ maxUInt64) : + parseDec (natToDecChars n) = some n := + Http.parseDec_printDec n h + +theorem parseHex_printHex (n : Nat) (h : n ≤ maxUInt64) : + parseHex (natToHexLower n) = some n := + Http.parseHex_printHex n h + +theorem determineFraming_disjoint (v : Version) (hdrs : List Header) (f1 f2 : Framing) + (h1 : determineFraming v hdrs = some f1) (h2 : determineFraming v hdrs = some f2) : + f1 = f2 := + Http.determineFraming_disjoint v hdrs f1 f2 h1 h2 + +theorem determineFraming_rejects_cl_and_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hcl : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = clNameLower) : + determineFraming v hdrs = none := + Http.determineFraming_rejects_cl_and_te v hdrs hte hcl + +theorem determineFraming_rejects_http10_te (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) : + determineFraming .http10 hdrs = none := + Http.determineFraming_rejects_http10_te hdrs hte + +theorem determineFraming_rejects_nonfinal_chunked (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ (∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower) ∧ + normalizedCodings hdrs = some [chunkedToken] := + Http.determineFraming_rejects_nonfinal_chunked v hdrs h + +theorem chunked_requires_http11_without_cl (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower := + Http.chunked_requires_http11_without_cl v hdrs h + +theorem determineFraming_rejects_bad_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hbad : normalizedCodings hdrs ≠ some [chunkedToken]) : + determineFraming v hdrs = none := + Http.determineFraming_rejects_bad_te v hdrs hte hbad + +theorem decodeChunked_encodeChunked (body : List UInt8) (h : body.length ≤ maxUInt64) : + decodeChunked (encodeChunked body) = some (body, [], []) := + Http.decodeChunked_encodeChunked body h + +theorem decodeChunked_prefix_free (buf body rest extra : List UInt8) (trailers : List Header) + (h : decodeChunked buf = some (body, trailers, rest)) : + decodeChunked (buf ++ extra) = some (body, trailers, rest ++ extra) := + Http.decodeChunked_prefix_free buf body rest extra trailers h + +theorem parseRequest_serializeFixed (hd : RequestHead) (body wire : List UInt8) + (hs : serializeFixed hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = hd.version ∧ + r.body = body ∧ r.rest = [] := + Http.parseRequest_serializeFixed hd body wire hs + +theorem parseRequest_serializeChunked (hd : RequestHead) (body wire : List UInt8) + (hs : serializeChunked hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = .http11 ∧ + r.body = body ∧ r.rest = [] := + Http.parseRequest_serializeChunked hd body wire hs + +theorem parseRequest_prefix_free (stream extra : List UInt8) (r : FramedRequest) + (h : parseRequest stream = some r) : + parseRequest (stream ++ extra) = some { r with rest := r.rest ++ extra } := + Http.parseRequest_prefix_free stream extra r h + +theorem parseRequest_stream_unique (stream tail1 tail2 : List UInt8) + (msgs1 msgs2 : List (RequestHead × List UInt8)) + (hlen : msgs1.length = msgs2.length) + (h1 : StreamFrames msgs1 tail1 stream) + (h2 : StreamFrames msgs2 tail2 stream) : + msgs1 = msgs2 ∧ tail1 = tail2 := + Http.parseRequest_stream_unique stream tail1 tail2 msgs1 msgs2 hlen h1 h2 + +theorem reject_smuggling_vectors : + ∀ v, v ∈ smugglingVectors → parseRequest v = none := + Http.reject_smuggling_vectors + +end Http.Spec diff --git a/Http/Spec.lean b/Http/Spec.lean new file mode 100644 index 0000000..9356a9a --- /dev/null +++ b/Http/Spec.lean @@ -0,0 +1,100 @@ +import Http.Lemmas + +namespace Http.Spec + +open Http + +theorem splitCRLF_sound (bs line rest : List UInt8) : + splitCRLF bs = some (line, rest) ↔ + NoBareLineBreak line ∧ bs = line ++ [0x0D, 0x0A] ++ rest := by + simpa [NoBareLineBreak, noBareLineBreak] using Http.splitCRLF_sound bs line rest + +theorem splitCRLF_prefix_free (bs line rest extra : List UInt8) + (h : splitCRLF bs = some (line, rest)) : + splitCRLF (bs ++ extra) = some (line, rest ++ extra) := + Http.splitCRLF_prefix_free bs line rest extra h + +theorem parseDec_printDec (n : Nat) (h : n ≤ maxUInt64) : + parseDec (natToDecChars n) = some n := + Http.parseDec_printDec n h + +theorem parseHex_printHex (n : Nat) (h : n ≤ maxUInt64) : + parseHex (natToHexLower n) = some n := + Http.parseHex_printHex n h + +theorem determineFraming_disjoint (v : Version) (hdrs : List Header) (f1 f2 : Framing) + (h1 : determineFraming v hdrs = some f1) (h2 : determineFraming v hdrs = some f2) : + f1 = f2 := + Http.determineFraming_disjoint v hdrs f1 f2 h1 h2 + +theorem determineFraming_rejects_cl_and_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hcl : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = clNameLower) : + determineFraming v hdrs = none := + Http.determineFraming_rejects_cl_and_te v hdrs hte hcl + +theorem determineFraming_rejects_http10_te (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) : + determineFraming .http10 hdrs = none := + Http.determineFraming_rejects_http10_te hdrs hte + +/-- Chunked framing is HTTP/1.1, with no Content-Length, and the combined +Transfer-Encoding list is exactly `[chunked]` after empty elements are dropped. -/ +theorem determineFraming_rejects_nonfinal_chunked (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ (∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower) ∧ + normalizedCodings hdrs = some [chunkedToken] := + Http.determineFraming_rejects_nonfinal_chunked v hdrs h + +theorem chunked_requires_http11_without_cl (v : Version) (hdrs : List Header) + (h : determineFraming v hdrs = some .chunked) : + v = .http11 ∧ ∀ hdr, hdr ∈ hdrs → lowerBytes hdr.name ≠ clNameLower := + Http.chunked_requires_http11_without_cl v hdrs h + +theorem determineFraming_rejects_bad_te (v : Version) (hdrs : List Header) + (hte : ∃ h, h ∈ hdrs ∧ lowerBytes h.name = teNameLower) + (hbad : normalizedCodings hdrs ≠ some [chunkedToken]) : + determineFraming v hdrs = none := + Http.determineFraming_rejects_bad_te v hdrs hte hbad + +theorem decodeChunked_encodeChunked (body : List UInt8) (h : body.length ≤ maxUInt64) : + decodeChunked (encodeChunked body) = some (body, [], []) := + Http.decodeChunked_encodeChunked body h + +theorem decodeChunked_prefix_free (buf body rest extra : List UInt8) (trailers : List Header) + (h : decodeChunked buf = some (body, trailers, rest)) : + decodeChunked (buf ++ extra) = some (body, trailers, rest ++ extra) := + Http.decodeChunked_prefix_free buf body rest extra trailers h + +theorem parseRequest_serializeFixed (hd : RequestHead) (body wire : List UInt8) + (hs : serializeFixed hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = hd.version ∧ + r.body = body ∧ r.rest = [] := + Http.parseRequest_serializeFixed hd body wire hs + +theorem parseRequest_serializeChunked (hd : RequestHead) (body wire : List UInt8) + (hs : serializeChunked hd body = some wire) : + ∃ r, parseRequest wire = some r ∧ r.head.method = hd.method ∧ + r.head.target = hd.target ∧ r.head.version = .http11 ∧ + r.body = body ∧ r.rest = [] := + Http.parseRequest_serializeChunked hd body wire hs + +theorem parseRequest_prefix_free (stream extra : List UInt8) (r : FramedRequest) + (h : parseRequest stream = some r) : + parseRequest (stream ++ extra) = some { r with rest := r.rest ++ extra } := + Http.parseRequest_prefix_free stream extra r h + +theorem parseRequest_stream_unique (stream tail1 tail2 : List UInt8) + (msgs1 msgs2 : List (RequestHead × List UInt8)) + (hlen : msgs1.length = msgs2.length) + (h1 : StreamFrames msgs1 tail1 stream) + (h2 : StreamFrames msgs2 tail2 stream) : + msgs1 = msgs2 ∧ tail1 = tail2 := + Http.parseRequest_stream_unique stream tail1 tail2 msgs1 msgs2 hlen h1 h2 + +theorem reject_smuggling_vectors : + ∀ v, v ∈ smugglingVectors → parseRequest v = none := + Http.reject_smuggling_vectors + +end Http.Spec diff --git a/Http/Spec/Defs.lean b/Http/Spec/Defs.lean new file mode 100644 index 0000000..8066b8e --- /dev/null +++ b/Http/Spec/Defs.lean @@ -0,0 +1,31 @@ +import Http.Parse + +namespace Http.Spec + +/-- A line contains neither CR nor LF. This is the property itself, not an alias. -/ +def NoBareLineBreak (bs : List UInt8) : Prop := + ∀ b, b ∈ bs → b ≠ 0x0D ∧ b ≠ 0x0A + +inductive StreamFrames : List (RequestHead × List UInt8) → List UInt8 → List UInt8 → Prop where + | nil (tail : List UInt8) : StreamFrames [] tail tail + | cons {stream : List UInt8} {r : FramedRequest} {msgs : List (RequestHead × List UInt8)} + {tail : List UInt8} : + Http.parseRequest stream = some r → + StreamFrames msgs tail r.rest → + StreamFrames ((r.head, r.body) :: msgs) tail stream + +/-- The ten request-smuggling inputs from the review. Bytes are the ASCII of those messages. -/ +def smugglingVectors : List (List UInt8) := [ + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 54, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 48, 13, 10, 13, 10, 71], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 52, 13, 10, 13, 10, 48, 13, 10, 13, 10], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 32, 99, 104, 117, 110, 107, 101, 100, 44, 32, 103, 122, 105, 112, 13, 10, 13, 10, 48, 13, 10, 13, 10], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 48, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 48, 13, 10, 13, 10], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 53, 13, 10, 104, 101, 108, 108, 111, 88, 88, 48, 13, 10, 13, 10], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 32, 58, 32, 53, 13, 10, 13, 10, 49, 50, 51, 52, 53], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, 13, 10, 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 48, 13, 10, 13, 10], + [71, 69, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 10, 10], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 53, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 49, 48, 13, 10, 13, 10, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48], + [80, 79, 83, 84, 32, 47, 32, 72, 84, 84, 80, 47, 49, 46, 49, 13, 10, 72, 111, 115, 116, 58, 32, 101, 120, 97, 109, 112, 108, 101, 46, 99, 111, 109, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 43, 53, 13, 10, 13, 10, 49, 50, 51, 52, 53] +] + +end Http.Spec diff --git a/Http/Types.lean b/Http/Types.lean new file mode 100644 index 0000000..2897f41 --- /dev/null +++ b/Http/Types.lean @@ -0,0 +1,43 @@ +namespace Http + +inductive Version where + | http10 + | http11 + deriving DecidableEq, Repr + +structure Header where + name : List UInt8 + value : List UInt8 + deriving DecidableEq, Repr + +structure RequestHead where + method : List UInt8 + target : List UInt8 + version : Version + headers : List Header + deriving DecidableEq, Repr + +inductive Framing where + | fixed (len : Nat) + | chunked + deriving DecidableEq, Repr + +structure FramedRequest where + head : RequestHead + body : List UInt8 + rest : List UInt8 + deriving DecidableEq, Repr + +def maxUInt64 : Nat := 2 ^ 64 - 1 + +def teNameLower : List UInt8 := [116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, 100, 105, 110, 103] + +def clNameLower : List UInt8 := [99, 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, 103, 116, 104] + +def hostNameLower : List UInt8 := [104, 111, 115, 116] + +def chunkedToken : List UInt8 := [99, 104, 117, 110, 107, 101, 100] + +def connectName : List UInt8 := [67, 79, 78, 78, 69, 67, 84] + +end Http diff --git a/Main.lean b/Main.lean new file mode 100644 index 0000000..096aba6 --- /dev/null +++ b/Main.lean @@ -0,0 +1,18 @@ +import Http.Parse + +open Http + +/-- Read one request from a file and print `accept N` or `reject`. -/ +def main : List String → IO UInt32 + | [] => do + IO.eprintln "usage: http-tool FILE" + return 2 + | path :: _ => do + let data ← IO.FS.readBinFile path + match parseRequest data.toList with + | none => + IO.println "reject" + return 0 + | some r => + IO.println s!"accept {r.body.length}" + return 0 diff --git a/README.md b/README.md index 955e541..552193c 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,86 @@ -# http: HTTP/1.1 request framing and chunked decoding, proved +# http: HTTP/1.1 framing and chunked bodies -This library specifies, frames, and serializes HTTP/1.x request messages in Lean 4 following RFC 9112 (Sections 2, 3, 5, 6, and 7.1), targeting machine-checked theorems for prefix-free stream framing and rejection of HTTP request-boundary ambiguities (`CL.TE`, `TE.CL`, `TE.TE`, `HTTP/1.0` `Transfer-Encoding`, and malformed chunk terminators). +This library reads one HTTP/1.x request from a byte string. It parses the head, refuses a head that breaks the rules in `SPEC.md`, and frames the body by a length or by the chunked coding. It then decodes a chunked body, including chunk extensions and the trailer section. -Status: in progress. The specification has not yet been approved by an independent reviewer on the org's list, so this is not yet verified. +The head rules are the same choices as the request-head parser: the four target forms, one Host on HTTP/1.1, no empty host, and CONNECT with a real port and no content. The body rules are RFC 9112 section 7. Where the RFC lets a recipient reject or repair, this library rejects. -## Planned and Drafted Theorems (`Http.Spec` / `Http.Proofs`) +It is a project of [Proven Commons](https://github.com/Proven-Commons/mission), built to the verified bar published there. + +Status: in progress. The specification has not yet been approved by a reviewer apart from its author, so this is not verified. + +## Use it + +``` +lake build +.lake/build/bin/http-tool request.bin +``` + +The tool prints `accept N` with the body length, or `reject`. From Lean, `Http.parseRequest` returns the head, the body, and the octets that follow. + +## What is proved + +The statements are in `Http/Spec.lean`. The proofs are in `Http/Proofs.lean`, under the same names. Definitions shared with the statements are in `Http/Spec/Defs.lean` and `Http/Types.lean`. | Claim | Theorem | -| :--- | :--- | -| `splitCRLF` splits at the earliest `CRLF` with no bare `CR` or `LF` in the line. | `Http.Spec.splitCRLF_sound` | -| `splitCRLF` is prefix-free when trailing pipelined bytes are appended. | `Http.Spec.splitCRLF_prefix_free` | -| Decimal formatting of numbers below `2^64` parses back to the same number. | `Http.Spec.parseDec_printDec` | -| Hexadecimal formatting of numbers below `2^64` parses back to the same number. | `Http.Spec.parseHex_printHex` | -| Body framing resolution assigns at most one framing mode to any header list. | `Http.Spec.determineFraming_disjoint` | -| Requests carrying both `Transfer-Encoding` and `Content-Length` are always rejected. | `Http.Spec.determineFraming_rejects_cl_and_te` | -| `HTTP/1.0` requests carrying `Transfer-Encoding` are always rejected (EEF-CVE-2026-94194). | `Http.Spec.determineFraming_rejects_http10_te` | -| Chunked framing requires `HTTP/1.1` and zero `Content-Length` headers. | `Http.Spec.determineFraming_rejects_nonfinal_chunked` | -| `encodeChunked` round-trips through `decodeChunked` with empty remainder. | `Http.Spec.decodeChunked_encodeChunked` | -| `decodeChunked` is prefix-free across pipelined TCP byte streams. | `Http.Spec.decodeChunked_prefix_free` | -| Canonical fixed-length request serialization round-trips through `parseRequest`. | `Http.Spec.parseRequest_serializeFixed` | -| Canonical chunked request serialization round-trips through `parseRequest`. | `Http.Spec.parseRequest_serializeChunked` | -| `parseRequest` is prefix-free when pipelined bytes follow on the TCP stream. | `Http.Spec.parseRequest_prefix_free` | -| Multi-message TCP streams decompose uniquely under `StreamFrames`. | `Http.Spec.parseRequest_stream_unique` | -| Normative RFC 9112 / CVE boundary-ambiguity vectors are rejected. | `Http.Spec.reject_smuggling_vectors` | +|-------|---------| +| A split at CRLF is a line with no bare CR or LF, the two octets `0x0D 0x0A`, and the suffix. | `Http.Spec.splitCRLF_sound` | +| Appending octets after a successful split keeps the line and appends those octets to the suffix. | `Http.Spec.splitCRLF_prefix_free` | +| Printing a length up to `2^64 - 1` in decimal and reading it back gives the same number. | `Http.Spec.parseDec_printDec` | +| Printing a length up to `2^64 - 1` in hex and reading it back gives the same number. | `Http.Spec.parseHex_printHex` | +| A version and a field list determine at most one framing. | `Http.Spec.determineFraming_disjoint` | +| Transfer-Encoding together with Content-Length is rejected. | `Http.Spec.determineFraming_rejects_cl_and_te` | +| Transfer-Encoding on HTTP/1.0 is rejected. | `Http.Spec.determineFraming_rejects_http10_te` | +| Chunked framing implies HTTP/1.1, no Content-Length field, and a normalized Transfer-Encoding list of exactly `[chunked]`. Empty elements are dropped before that comparison, so the list is final and is not repeated. Case variants are normalized to lower case, and field lines are joined before the list is split. | `Http.Spec.determineFraming_rejects_nonfinal_chunked` | +| The same hypothesis implies HTTP/1.1 and no Content-Length, which is the weaker form of the previous statement. | `Http.Spec.chunked_requires_http11_without_cl` | +| A Transfer-Encoding field whose normalized list is anything other than `[chunked]` is rejected, including no coding, a repeated coding, and a coding that is not final. | `Http.Spec.determineFraming_rejects_bad_te` | +| Encoding a body whose length is at most `2^64 - 1` and decoding it gives that body, no trailers, and an empty suffix. | `Http.Spec.decodeChunked_encodeChunked` | +| Appending octets after a successful chunked body keeps the body and the trailers and appends those octets to the suffix. | `Http.Spec.decodeChunked_prefix_free` | +| A request printed with a length framing parses back to the same method, target, version, and body. | `Http.Spec.parseRequest_serializeFixed` | +| A request printed with chunked framing parses back to the same method and target, version HTTP/1.1, and the same body. | `Http.Spec.parseRequest_serializeChunked` | +| Appending octets after a successfully parsed request appends those octets to its suffix. | `Http.Spec.parseRequest_prefix_free` | +| Two ways of splitting one byte string into the same number of requests produce the same requests and the same tail. | `Http.Spec.parseRequest_stream_unique` | +| The ten smuggling messages in `SPEC.md` are rejected. | `Http.Spec.reject_smuggling_vectors` | + +`Http.Spec.determineFraming_rejects_nonfinal_chunked` is the strong statement. It includes `normalizedCodings hdrs = some [chunkedToken]`. `Http.Spec.chunked_requires_http11_without_cl` keeps the weaker pair of facts on its own. ## What is not proved -- Hex CLI decoding/encoding in `Main.lean` (covered by differential tests). -- HTTP response framing (`1xx`, `204`, `304`, and `HEAD` bodyless response rules in RFC 9112 Section 6.3). -- Content-coding decompression (`gzip`, `deflate`, `br`) and TLS / socket I/O. +- The command-line tool in `Main.lean`: reading the file and printing the line. The tests run it. +- That every refusal the RFCs require is one of these refusals. `SPEC.md` says which rule each refusal comes from. +- The IPv6 grammar inside brackets, and the full `absolute-URI` grammar for a scheme other than http or https. `SPEC.md` delimits both, and both belong to the request-head parser. +- Trailer fields after they have been parsed. The decoder checks their shape and drops them. +- What a server does with a request it accepts, and the response it sends after a refusal. +- A bound on the size of a message. The chunked decoder stops after a number of steps linear in the input length; that fuel is part of the function, and the round-trip theorems use a body that fits in one chunk. + +## What you must trust + +- The Lean kernel, which checks the proofs. +- The Lean compiler, which turns the definitions into the program that runs, and the runtime of that program. +- The operating system and the hardware it runs on. +- `lake comparator`, which checks that each proof in `Http/Proofs.lean` proves the statement of the same name in `Http/Spec.lean`. +- The transcription of the rules in `SPEC.md`. A reviewer checks it against RFC 9112 and RFC 9110. The tests compare the tool with a second transcription of those rules. + +## Re-check + +Needs Linux, [elan](https://github.com/leanprover/elan), bubblewrap (`apt install bubblewrap`), and Python 3. + +``` +lake build +lake comparator --paranoid +python3 tests/run_tests.py +``` + +`lake comparator --paranoid` builds the statements and the proofs in a sandbox and checks that each proof proves its statement, using only `propext`, `Classical.choice`, and `Quot.sound`. + +`python3 tests/run_tests.py` feeds requests to `http-tool` and compares each answer with the Python transcription of `SPEC.md`. It rewrites `tests/TEST-RESULTS.md`. + +## Layout + +- `Http/Types.lean`: versions, fields, and framing. It is part of the specification. +- `Http/Bytes.lean`, `Http/Framing.lean`, `Http/Parse.lean`: the code under proof. +- `Http/Lemmas.lean`: the proofs of the statements, which `Http/Spec.lean` and `Http/Proofs.lean` both cite. +- `Http/Spec/Defs.lean`: the line property, the stream relation, and the ten smuggling messages. +- `Http/Spec.lean`: the statements. A reviewer approves this file. +- `Http/Proofs.lean`: the same statements, proved. +- `Main.lean`: the command-line tool. +- `SPEC.md`: the rules, including the Host and CONNECT choices shared with the request-head parser. diff --git a/SPEC.md b/SPEC.md index 608fcae..e5cc0a2 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,22 +1,81 @@ -# HTTP/1.1 Message Framing and Chunked Transfer Coding (`Http`) - -This specification defines the wire-level rules for parsing and framing an HTTP/1.x request message from a byte stream under RFC 9112 (HTTP/1.1) and RFC 9110 (HTTP Semantics), taking the strict reading wherever lenient parsers permit request-boundary disagreement. - -## Scope - -1. **Line Termination (`splitCRLF`)**: Every start-line, header field line, chunk-size line, chunk-data record, and trailer section MUST end with `CRLF` (`0x0D 0x0A`). Any bare `CR` (`0x0D` not followed by `0x0A`) or bare `LF` (`0x0A` not preceded by `0x0D`) inside a line is rejected (`none`). -2. **Request-Line (`parseRequestLine`, RFC 9112 Section 3)**: Must split on single `SP` (`0x20`) into three components `method SP request-target SP HTTP-version`. Runs of multiple spaces, `HTAB`, or control characters are rejected. `method` is `1*tchar`; `request-target` is `1*VCHAR`; `HTTP-version` is `HTTP/1.1` or `HTTP/1.0`. -3. **Header Section (`parseHeaders`, RFC 9112 Section 5)**: - - Obsolete line folding (`obs-fold`, lines beginning with `SP` or `HTAB`, Section 5.2) is rejected. - - Whitespace between `field-name` and `:` (Section 5.1) is rejected; `field-name` must satisfy `1*tchar`. - - Leading and trailing optional whitespace (`OWS`: `SP` / `HTAB`) around `field-value` is stripped; interior bytes must satisfy RFC 9110 Section 5.5 (`field-vchar` or `OWS`, no `NUL` or control octets). -4. **Message Body Framing (`determineFraming`, RFC 9112 Sections 6.1-6.3)**: - - **Combined `Transfer-Encoding` and `Content-Length` (Sections 6.1 / 6.3)**: Unconditionally rejected to prevent `CL.TE` and `TE.CL` boundary ambiguity. - - **`Transfer-Encoding` in `HTTP/1.0` (Section 6.1, EEF-CVE-2026-94194)**: Unconditionally rejected. - - **Transfer-Coding List (Sections 6.1 / 6.3, EEF-CVE-2026-94194)**: Tokenized across comma-separated values; empty elements, non-token characters, repeated `chunked`, or non-final `chunked` (`Transfer-Encoding: chunked, gzip`) are rejected. Only `[chunked]` (case-insensitively) selects chunked framing in this slice. - - **`Content-Length` (Section 6.2 / RFC 9110 Section 8.6)**: Duplicate `Content-Length` header fields are rejected; a single `Content-Length` value must match `1*DIGIT` (`< 2^64`, no `+`, `-`, or comma list). - - **Default**: Requests with neither `Transfer-Encoding` nor `Content-Length` have a zero-byte body (`fixed 0`). -5. **Chunked Transfer Coding (`decodeChunked`, RFC 9112 Section 7.1)**: - - `chunk-size` is `1*HEXDIG` (`< 2^64`). Optional chunk extensions after `;` must contain only `OWS` and `field-vchar`. - - Every non-zero chunk payload of `sz` bytes MUST be followed immediately by `0x0D 0x0A` (`CRLF`); non-CRLF terminators (CVE-2025-23167 class) are rejected. - - The terminating `0` chunk is followed by an optional RFC 9112 Section 7.1.2 trailer section (`*( field-line CRLF )`) and final `CRLF`. +# HTTP/1.1 request framing and chunked bodies + +This specification is the framing half of an HTTP/1.1 request parser. It reads one request from a byte string: the head, then a body framed by a length or by the chunked coding, and the octets that follow. It is written to agree with the request-head rules of the companion parser (the four target forms, Host, and CONNECT), and then to decode the body those rules frame. + +Where RFC 9112 lets a recipient reject or repair, this library rejects. Field names are compared without regard to ASCII case. + +Status: in progress. + +## What this library checks + +### Lines + +Every request line, field line, chunk-size line, and trailer line ends in CRLF. A bare CR or a bare LF in that line is rejected. Empty lines before the request line are skipped. An empty line ends the header section, and another ends the trailer section. + +### Request line + +The request line is `method SP request-target SP HTTP-version`, split on a single SP into three components. The method is a token (`1*tchar`). The version is `HTTP/1.0`, or `HTTP/1.1` through `HTTP/1.9`. A minor version of 1 or more is HTTP/1.1 for every rule below. Any other version is rejected. + +### The four target forms + +The target is one of the four forms of RFC 9112 section 3.2, checked as a stated subset of RFC 3986. It is a string of VCHARs, and `#` is rejected in every form, so `GET /#frag` is refused. + +- **origin-form** begins with `/`. +- **asterisk-form** is exactly `*`. +- **absolute-form** for the schemes `http` and `https`, in any case, is `( "http" / "https" ) "://" host [ ":" port ] path-abempty [ "?" query ]` with a host that is not empty and no userinfo. `http://a/x` and `HTTP://a:/x` are accepted. `http:`, `http:/a`, `http://`, `http:///a`, and `https://u@a/` are rejected. An empty port is allowed. +- **any other scheme** is `scheme ":" 1*VCHAR` with no `#`, where `scheme` is an ALPHA followed by ALPHA, DIGIT, `+`, `-`, or `.`. This is not the full `absolute-URI` grammar. The caller that needs that grammar is the request-head parser. +- **authority-form** is what remains: `host [ ":" port ]` with no `/` and no `?`. The host is not empty. A target that begins with `:`, such as `GET :80` or `OPTIONS :`, is refused for every method: it is an authority-form target with an empty host, and RFC 9112 section 3.3 plus RFC 9110 section 4.2.1 require that URI to be rejected. + +A host here is either a run of VCHARs with no colon, or a bracketed literal: `[` then at least one octet then `]` then an optional `:` and a port of digits. The port may be empty. **The octets inside the brackets are not matched against the IPv6 grammar of RFC 3986.** That match is the request-head parser's derivative matcher, and this library does not repeat it. `CONNECT [::1]:443` and `Host: [::1]` are accepted here because the brackets are nonempty and the port is digits, not because `::1` was proved to be an IPv6 address. + +### Host + +A request with a minor version of 1 or more needs exactly one Host field. Any other request needs at most one. The value is empty, or a host that is not empty with an optional port, so `Host: :80` and `Host: :` are refused and an empty Host is accepted. An invalid Host value is refused. This is the same reading as the request-head parser: RFC 9112 section 3.2 for the count, and sections 3.2 and 3.3 together with RFC 9110 section 4.2.1 for an empty host before a colon. + +### CONNECT + +When the method is exactly `CONNECT` (the method token is case-sensitive, so `connect` is a different method): + +- the target is `uri-host ":" port` with a nonempty host and a port from 1 to 65535, leading zeros allowed, so `a:00080` is port 80; +- an empty port, port 0, a port above 65535, and a target that is not a host and port (`/`, `*`, `:80`) are refused; +- the request has no content. Transfer-Encoding, or a Content-Length that is not numerically 0, is refused. `Content-Length: 0` and `Content-Length: 000` are accepted. RFC 9110 section 9.3.6 says a CONNECT request does not have content, and a server need not accept CONNECT at all. + +`CONNECT http:80` is the host `http` and the port 80. It is not an http URI. `connect http:80`, with the method in lower case, is held to the http-target rule and refused. + +### Framing + +`determineFraming` looks only at the version and the fields. + +Transfer-Encoding field lines are one combined value: their values in order, joined with commas. That value is split on commas, each element is trimmed of OWS (SP or HTAB), and empty elements are dropped. Empty elements do not count. `, chunked`, `chunked,`, and a `chunked` line beside an empty Transfer-Encoding line are the coding `chunked`. A tab or spaces around the token are OWS and are trimmed. Each remaining element must be a bare token, so `chunked;a=b` is refused. + +Chunked framing is returned only when the version is HTTP/1.1 (or a later 1.x), there is no Content-Length field, and the normalized list is exactly `[chunked]`. In particular: + +- no coding (`Transfer-Encoding:` or `Transfer-Encoding: ,`) is rejected; +- chunked not last (`chunked, gzip`, `gzip, chunked`) is rejected; +- chunked repeated (`chunked, chunked`, or `chunked` on two lines) is rejected; +- Transfer-Encoding together with Content-Length is rejected; +- Transfer-Encoding on HTTP/1.0 is rejected. + +Content-Length is accepted only as one field whose value is `1*DIGIT` and whose number is at most `2^64 - 1`. It frames that many octets. With neither Content-Length nor Transfer-Encoding, the body has length 0. Two Content-Length lines are rejected even when they agree. A sign, a comma, or any other non-digit is rejected. + +### Chunked bodies + +`chunk-size` is `1*HEXDIG`, at most `2^64 - 1`. The size is the hex number, and what follows it on the line is a chunk extension, not ignored whitespace. + +A chunk extension is the grammar of RFC 9112 section 7.1.1, not a whitelist of field characters: `;` OWS `token` [OWS `=` OWS (`token` / quoted-string)], and the extension may repeat. `token` is `1*tchar`. A quoted-string is the usual quoted text, with a backslash escape of HTAB, SP, VCHAR, or an octet at least 128. This library accepts a well-formed extension it does not understand, including a quoted value such as `1;a="x"`. It rejects a missing name (`1;=x`) and a quote that never closes (`1;a="x`). + +Each non-zero chunk is that many octets followed by CRLF. Any other terminator is rejected. The zero chunk is followed by a trailer section of the same shape as the header section, then the octets after the trailer. The trailer is parsed and then discarded: this library does not interpret trailer fields. + +## What is delegated + +These checks belong to the request-head parser, and this specification does not claim them: + +- the RFC 3986 IPv6 and IPvFuture grammar inside brackets; +- the full `absolute-URI` grammar for schemes other than the shape above; +- percent-encoding, and what a proxy does after it decodes a target; +- a limit on the size of a head; +- the response a server sends after a refusal. + +## Smuggling inputs + +The ten messages in `Http.Spec.smugglingVectors` are rejected. They are Content-Length with Transfer-Encoding, Transfer-Encoding with Content-Length, `chunked, gzip`, Transfer-Encoding on HTTP/1.0, a chunk whose terminator is not CRLF, a space before a colon, a folded line, a bare LF, two Content-Length lines, and `Content-Length: +5`. diff --git a/bar.toml b/bar.toml index 444c92a..934b5a8 100644 --- a/bar.toml +++ b/bar.toml @@ -7,9 +7,23 @@ entry_points = ["Main.lean"] [spec] files = ["SPEC.md", "Http/Types.lean", "Http/Spec/Defs.lean", "Http/Spec.lean"] -code = ["Http.Types", "Http.Framing", "Http"] +code = [ + "Http.splitCRLF", + "Http.parseDec", + "Http.natToDecChars", + "Http.parseHex", + "Http.natToHexLower", + "Http.determineFraming", + "Http.normalizedCodings", + "Http.lowerBytes", + "Http.decodeChunked", + "Http.encodeChunked", + "Http.serializeFixed", + "Http.serializeChunked", + "Http.parseRequest", +] [tests] results = "tests/TEST-RESULTS.md" -command = ["python3", "tests/run_tests.py"] -files = ["tests/run_tests.py", "tests/planted_bugs.py", "tests/PLANTED-BUGS.md"] +command = "python3 tests/run_tests.py" +files = ["tests/run_tests.py"] diff --git a/comparator.json b/comparator.json new file mode 100644 index 0000000..b34def1 --- /dev/null +++ b/comparator.json @@ -0,0 +1,28 @@ +{ + "challenge_module": "Http.Spec", + "solution_module": "Http.Proofs", + "theorem_names": [ + "Http.Spec.splitCRLF_sound", + "Http.Spec.splitCRLF_prefix_free", + "Http.Spec.parseDec_printDec", + "Http.Spec.parseHex_printHex", + "Http.Spec.determineFraming_disjoint", + "Http.Spec.determineFraming_rejects_cl_and_te", + "Http.Spec.determineFraming_rejects_http10_te", + "Http.Spec.determineFraming_rejects_nonfinal_chunked", + "Http.Spec.chunked_requires_http11_without_cl", + "Http.Spec.determineFraming_rejects_bad_te", + "Http.Spec.decodeChunked_encodeChunked", + "Http.Spec.decodeChunked_prefix_free", + "Http.Spec.parseRequest_serializeFixed", + "Http.Spec.parseRequest_serializeChunked", + "Http.Spec.parseRequest_prefix_free", + "Http.Spec.parseRequest_stream_unique", + "Http.Spec.reject_smuggling_vectors" + ], + "permitted_axioms": [ + "propext", + "Quot.sound", + "Classical.choice" + ] +} diff --git a/lake-manifest.json b/lake-manifest.json new file mode 100644 index 0000000..e5fecfa --- /dev/null +++ b/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.3.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "http", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/lakefile.toml b/lakefile.toml new file mode 100644 index 0000000..f5ec55f --- /dev/null +++ b/lakefile.toml @@ -0,0 +1,19 @@ +name = "http" +version = "0.1.0" +license = "Apache-2.0" +defaultTargets = ["Http", "http-tool"] + +[[lean_lib]] +name = "Http" + +[[lean_lib]] +name = "HttpSpec" +roots = ["Http.Spec"] + +[[lean_lib]] +name = "HttpProofs" +roots = ["Http.Proofs"] + +[[lean_exe]] +name = "http-tool" +root = "Main" diff --git a/lean-toolchain b/lean-toolchain new file mode 100644 index 0000000..a6aff06 --- /dev/null +++ b/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.35.0 diff --git a/tests/TEST-RESULTS.md b/tests/TEST-RESULTS.md new file mode 100644 index 0000000..c9480ba --- /dev/null +++ b/tests/TEST-RESULTS.md @@ -0,0 +1,6 @@ +# Test record + +- Oracle: Python transcription of SPEC.md, compared with http-tool +- Seed: 20261120 +- Inputs: 30380 +- Failures: 0 diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 0000000..5263e1c --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +"""Compare http-tool with a second transcription of SPEC.md.""" + +import pathlib +import random +import subprocess +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] +BIN = ROOT / ".lake" / "build" / "bin" / "http-tool" +SEED = 20261120 +MAX = (1 << 64) - 1 +RESULTS = ROOT / "tests" / "TEST-RESULTS.md" + +TCHAR = set(b"!#$%&'*+-.^_`|~") + + +def is_vchar(b): + return 33 <= b <= 126 + + +def is_tchar(b): + return 65 <= b <= 90 or 97 <= b <= 122 or 48 <= b <= 57 or b in TCHAR + + +def is_digit(b): + return 48 <= b <= 57 + + +def is_hex(b): + return is_digit(b) or 97 <= b <= 102 or 65 <= b <= 70 + + +def is_alpha(b): + return 65 <= b <= 90 or 97 <= b <= 122 + + +def lower(bs): + return bytes(b + 32 if 65 <= b <= 90 else b for b in bs) + + +def split_crlf(bs): + for i, b in enumerate(bs): + if b == 13: + if i + 1 < len(bs) and bs[i + 1] == 10: + return bs[:i], bs[i + 2 :] + return None + if b == 10: + return None + return None + + +def skip_empty(bs): + while bs.startswith(b"\r\n"): + bs = bs[2:] + return bs + + +def split_byte(sep, bs): + parts = [bytearray()] + for b in bs: + if b == sep: + parts.append(bytearray()) + else: + parts[-1].append(b) + return [bytes(p) for p in parts] + + +def split_first(sep, bs): + for i, b in enumerate(bs): + if b == sep: + return bs[:i], bs[i + 1 :] + return None + + +def parse_acc(base, val, acc, bs): + for b in bs: + d = val(b) + if d is None: + return None + acc = acc * base + d + if acc > MAX: + return None + return acc + + +def dec_val(b): + return b - 48 if 48 <= b <= 57 else None + + +def hex_val(b): + if 48 <= b <= 57: + return b - 48 + if 97 <= b <= 102: + return b - 87 + if 65 <= b <= 70: + return b - 55 + return None + + +def parse_dec(bs): + if not bs: + return None + return parse_acc(10, dec_val, 0, bs) + + +def parse_hex(bs): + if not bs: + return None + return parse_acc(16, hex_val, 0, bs) + + +def digits(bs): + return bool(bs) and all(is_digit(b) for b in bs) + + +def vchars(bs): + return all(is_vchar(b) for b in bs) + + +def bare_token(bs): + return bool(bs) and all(is_tchar(b) for b in bs) + + +def trim_ows(bs): + i = 0 + while i < len(bs) and bs[i] in (32, 9): + i += 1 + j = len(bs) + while j > i and bs[j - 1] in (32, 9): + j -= 1 + return bs[i:j] + + +def host_port(bs): + if not bs or not vchars(bs): + return False + if bs[0] == 91: + cut = split_first(93, bs) + if cut is None: + return False + lit, rest = cut + if len(lit) <= 1: + return False + if not rest: + return True + if rest[0] != 58: + return False + port = rest[1:] + return not port or digits(port) + cut = split_first(58, bs) + if cut is None: + return True + host, port = cut + return bool(host) and (not port or digits(port)) + + +def http_target(bs): + low = lower(bs) + if low.startswith(b"https:"): + pre = 6 + elif low.startswith(b"http:"): + pre = 5 + else: + return False + rest = bs[pre:] + if not rest.startswith(b"//"): + return False + auth_and = rest[2:] + cut = len(auth_and) + for i, b in enumerate(auth_and): + if b in (47, 63): + cut = i + break + auth, path = auth_and[:cut], auth_and[cut:] + return ( + vchars(bs) + and 64 not in auth + and host_port(auth) + and vchars(path) + and 35 not in path + ) + + +def scheme_rest(bs): + cut = split_first(58, bs) + if cut is None: + return False + scheme, rest = cut + if not scheme or not is_alpha(scheme[0]): + return False + if not all(is_alpha(b) or is_digit(b) or b in (43, 45, 46) for b in scheme[1:]): + return False + if not vchars(rest) or 35 in rest: + return False + return lower(scheme) not in (b"http", b"https") + + +def connect_port(port): + n = parse_dec(port) + return n is not None and digits(port) and 1 <= n <= 65535 + + +def connect_target(bs): + if not bs or not vchars(bs) or 64 in bs: + return False + if bs[0] == 91: + cut = split_first(93, bs) + if cut is None: + return False + _, rest = cut + return bool(rest) and rest[0] == 58 and connect_port(rest[1:]) + cut = split_first(58, bs) + if cut is None: + return False + host, port = cut + return bool(host) and 58 not in host and connect_port(port) + + +def target_ok(method, target): + if method == b"CONNECT": + return connect_target(target) + low = lower(target) + if low.startswith(b"http:") or low.startswith(b"https:"): + return http_target(target) + if target == b"*": + return True + if target[:1] == b"/": + return vchars(target) and 35 not in target + if scheme_rest(target): + return True + return host_port(target) and 47 not in target and 63 not in target and 35 not in target + + +def parse_version(bs): + if len(bs) != 8 or bs[:7] != b"HTTP/1.": + return None + d = bs[7] + if d == 48: + return "1.0" + if 49 <= d <= 57: + return "1.1" + return None + + +def parse_field(line): + if not line or line[0] in (32, 9): + return None + cut = split_first(58, line) + if cut is None: + return None + name, raw = cut + value = trim_ows(raw) + if bare_token(name) and all(b == 9 or b == 32 or is_vchar(b) or b >= 128 for b in value): + return name, value + return None + + +def parse_fields(bs): + headers = [] + fuel = len(bs) + while True: + if fuel == 0: + return None + fuel -= 1 + cut = split_crlf(bs) + if cut is None: + return None + line, bs = cut + if line == b"": + return headers, bs + field = parse_field(line) + if field is None: + return None + headers.append(field) + + +def host_value_ok(v): + return v == b"" or (host_port(v) and (not v or v[0] != 58)) + + +def te_values(headers): + return [v for n, v in headers if lower(n) == b"transfer-encoding"] + + +def cl_headers(headers): + return [(n, v) for n, v in headers if lower(n) == b"content-length"] + + +def normalized(headers): + vals = te_values(headers) + if not vals: + return [] + joined = b",".join(vals) + elems = [trim_ows(p) for p in joined.split(b",")] + elems = [e for e in elems if e] + if not all(bare_token(e) for e in elems): + return None + return [lower(e) for e in elems] + + +def cl_state(headers): + got = cl_headers(headers) + if not got: + return ("absent", None) + if len(got) != 1 or not digits(got[0][1]): + return ("bad", None) + n = parse_dec(got[0][1]) + if n is None: + return ("bad", None) + return ("ok", n) + + +def request_ok(method, target, version, headers): + hosts = [v for n, v in headers if lower(n) == b"host"] + if version == "1.1": + if len(hosts) != 1: + return False + elif len(hosts) > 1: + return False + if not all(host_value_ok(v) for v in hosts): + return False + if method == b"CONNECT": + if te_values(headers): + return False + kind, n = cl_state(headers) + if kind == "ok" and n != 0: + return False + if kind == "bad": + return False + if target[:1] == b":": + return False + return target_ok(method, target) + + +def framing(version, headers): + codes = normalized(headers) + if codes is None: + return None + kind, n = cl_state(headers) + if kind == "bad": + return None + if kind == "ok": + if codes == [] and not te_values(headers): + return ("fixed", n) + return None + if codes == [] and not te_values(headers): + return ("fixed", 0) + if codes == [b"chunked"] and version == "1.1": + return ("chunked", None) + return None + + +def is_qdtext(b): + return b in (9, 32, 33) or 35 <= b <= 91 or 93 <= b <= 126 or b >= 128 + + +def is_quoted_char(b): + return b in (9, 32) or is_vchar(b) or b >= 128 + + +def parse_quoted(bs): + if not bs: + return None + if bs[0] == 34: + return bs[1:] + if bs[0] == 92: + if len(bs) < 2 or not is_quoted_char(bs[1]): + return None + return parse_quoted(bs[2:]) + if is_qdtext(bs[0]): + return parse_quoted(bs[1:]) + return None + + +def skip_ows(bs): + i = 0 + while i < len(bs) and bs[i] in (32, 9): + i += 1 + return bs[i:] + + +def take_token(bs): + i = 0 + while i < len(bs) and is_tchar(bs[i]): + i += 1 + if i == 0: + return None + return bs[i:] + + +def parse_one_ext(bs): + bs = skip_ows(bs) + if not bs or bs[0] != 59: + return None + after = take_token(skip_ows(bs[1:])) + if after is None: + return None + rest = skip_ows(after) + if not rest or rest[0] != 61: + return rest + val = skip_ows(rest[1:]) + if val[:1] == b'"': + return parse_quoted(val[1:]) + return take_token(val) + + +def parse_ext(bs): + fuel = len(bs) + while bs: + if fuel == 0: + return False + fuel -= 1 + nxt = parse_one_ext(bs) + if nxt is None or len(nxt) >= len(bs): + return False + bs = nxt + return True + + +def parse_chunk_size(line): + i = 0 + while i < len(line) and is_hex(line[i]): + i += 1 + if i == 0: + return None + n = parse_hex(line[:i]) + if n is None or not parse_ext(line[i:]): + return None + return n + + +def parse_trailers(bs): + fuel = len(bs) + headers = [] + while True: + if fuel == 0: + return None + fuel -= 1 + cut = split_crlf(bs) + if cut is None: + return None + line, bs = cut + if line == b"": + return headers, bs + if parse_field(line) is None: + return None + headers.append(parse_field(line)) + + +def decode_chunked(buf): + fuel = len(buf) + 1 + acc = b"" + while True: + if fuel == 0: + return None + fuel -= 1 + cut = split_crlf(buf) + if cut is None: + return None + line, rest = cut + n = parse_chunk_size(line) + if n is None: + return None + if n == 0: + got = parse_trailers(rest) + if got is None: + return None + _, rest2 = got + return acc, rest2 + if len(rest) < n + 2 or rest[n : n + 2] != b"\r\n": + return None + acc += rest[:n] + buf = rest[n + 2 :] + + +def parse_request(bs): + cut = split_crlf(skip_empty(bs)) + if cut is None: + return None + line, rest = cut + parts = split_byte(32, line) + if len(parts) != 3: + return None + method, target, version_b = parts + version = parse_version(version_b) + if version is None or not bare_token(method) or not target or not vchars(target): + return None + got = parse_fields(rest) + if got is None: + return None + headers, rest = got + if not request_ok(method, target, version, headers): + return None + fr = framing(version, headers) + if fr is None: + return None + kind, n = fr + if kind == "fixed": + if len(rest) < n: + return None + return n + decoded = decode_chunked(rest) + if decoded is None: + return None + body, _ = decoded + return len(body) + + +def tool(data): + with tempfile.NamedTemporaryFile(delete=False) as fh: + fh.write(data) + path = fh.name + out = subprocess.run([str(BIN), path], check=False, capture_output=True) + text = out.stdout.decode() + if text == "reject\n": + return None + if text.startswith("accept "): + return int(text.split()[1]) + raise SystemExit(f"unexpected tool output {text!r} for {data!r}") + + +SMUGGLE = [ + b"POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: 6\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\nG", + b"POST / HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\nContent-Length: 4\r\n\r\n0\r\n\r\n", + b"POST / HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked, gzip\r\n\r\n0\r\n\r\n", + b"POST / HTTP/1.0\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", + b"POST / HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhelloXX0\r\n\r\n", + b"POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length : 5\r\n\r\n12345", + b"POST / HTTP/1.1\r\nHost: example.com\r\nTransfer-Encoding:\r\n chunked\r\n\r\n0\r\n\r\n", + b"GET / HTTP/1.1\nHost: example.com\n\n", + b"POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: 5\r\nContent-Length: 10\r\n\r\n1234567890", + b"POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: +5\r\n\r\n12345", +] + + +def chunked_body(payload, ext=b""): + size = format(len(payload), "x").encode() + return size + ext + b"\r\n" + payload + b"\r\n0\r\n\r\n" + + +def messages(): + methods = [b"GET", b"POST", b"OPTIONS", b"CONNECT", b"connect"] + targets = [ + b"/", + b"/a", + b"/a?q=1", + b"*", + b"*no", + b":80", + b":", + b"a:80", + b"a:00080", + b"a:0", + b"a:65536", + b"a:", + b"http://a/x", + b"HTTP://a:/x", + b"http://", + b"http:///a", + b"https://u@a/", + b"http:", + b"http:/a", + b"ftp://a", + b"/#frag", + b"[::1]:443", + b"[]:80", + ] + versions = [b"HTTP/1.0", b"HTTP/1.1", b"HTTP/1.2"] + header_sets = [ + [], + [b"Host: a"], + [b"Host:"], + [b"Host: :80"], + [b"Host: :"], + [b"Host: a", b"Host: b"], + [b"Host: [::1]"], + [b"Host: []"], + [b"Content-Length: 0"], + [b"Content-Length: 5"], + [b"Content-Length: 000"], + [b"Content-Length: +5"], + [b"Content-Length: 5", b"Content-Length: 5"], + [b"Transfer-Encoding: chunked"], + [b"Transfer-Encoding: Chunked"], + [b"Transfer-Encoding: chunked,"], + [b"Transfer-Encoding: , chunked"], + [b"Transfer-Encoding: chunked, gzip"], + [b"Transfer-Encoding: gzip, chunked"], + [b"Transfer-Encoding: chunked, chunked"], + [b"Transfer-Encoding: chunked;a=b"], + [b"Transfer-Encoding:"], + [b"Transfer-Encoding: ,"], + [b"Transfer-Encoding: chunked", b"Transfer-Encoding:"], + [b"Transfer-Encoding: chunked", b"Content-Length: 0"], + [b"Host: a", b"Content-Length: 5"], + [b"Host: a", b"Transfer-Encoding: chunked"], + [b"Host: a", b"Transfer-Encoding: , chunked"], + [b"Host: a", b"Transfer-Encoding: chunked,"], + [b"Host: a", b"Content-Length: 0"], + ] + bodies = [b"", b"hello", b"Z"] + exts = [b"", b";a=b", b';a="x"', b";=x", b';a="x'] + out = list(SMUGGLE) + for method in methods: + for target in targets: + for version in versions: + for headers in header_sets: + # A content-length of 5 is paired with a 5-octet body. + # Other bodies are appended only for chunked and empty. + head = method + b" " + target + b" " + version + b"\r\n" + for h in headers: + head += h + b"\r\n" + head += b"\r\n" + names = b" ".join(headers) + if b"chunked" in lower(names) and b"Content-Length" not in names: + for ext in exts: + out.append(head + chunked_body(b"Z", ext)) + elif b"Content-Length: 5" in names and b"Content-Length: 5\r" not in names + b"\r": + out.append(head + b"hello") + elif any(h == b"Content-Length: 5" for h in headers): + out.append(head + b"hello") + else: + out.append(head + b"") + if b"Content-Length" not in names and b"chunked" not in lower(names): + out.append(head + b"extra") + # A few explicit chunk and fold cases the grid does not build. + out.append(b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding:\tchunked\r\n\r\n" + chunked_body(b"Z")) + out.append(b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\nTransfer-Encoding: gzip\r\n\r\n0\r\n\r\n") + out.append(b"GET / HTTP/1.1\r\nHost: a\r\n\r\n") + out.append(b"\r\n\r\nGET / HTTP/1.1\r\nHost: a\r\n\r\n") + out.append(b"CONNECT a:443 HTTP/1.1\r\nHost: a\r\nContent-Length: 0\r\n\r\n") + out.append(b"CONNECT a:443 HTTP/1.1\r\nHost: a\r\nContent-Length: 5\r\n\r\nhello") + out.append(b"CONNECT a:443 HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n") + out.append(b"GET :80 HTTP/1.1\r\nHost: a\r\n\r\n") + out.append(b"OPTIONS : HTTP/1.1\r\nHost: a\r\n\r\n") + out.append(b"GET /#frag HTTP/1.1\r\nHost: a\r\n\r\n") + # Keep the unused samples referenced so a later edit can see them. + del bodies + return out + + +def main(): + if not BIN.is_file(): + raise SystemExit(f"missing {BIN}; run lake build first") + random.Random(SEED) + failures = [] + data = messages() + for i, msg in enumerate(data): + expect = parse_request(msg) + got = tool(msg) + if expect != got: + failures.append((i, expect, got, msg[:120])) + if len(failures) >= 12: + break + text = ( + "# Test record\n\n" + "- Oracle: Python transcription of SPEC.md, compared with http-tool\n" + f"- Seed: {SEED}\n" + f"- Inputs: {len(data)}\n" + f"- Failures: {len(failures)}\n" + ) + RESULTS.write_text(text) + if failures: + for i, expect, got, sample in failures: + print(f"mismatch {i}: oracle {expect} tool {got} sample {sample!r}") + raise SystemExit(1) + print(text) + + +if __name__ == "__main__": + main()