From ba700c5262307684de4d7e8ab6306a457ee0848c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 23:50:24 +0000 Subject: [PATCH 1/3] First slice: Go's path.Clean and path.Join, proved Path.clean is Go's loop, read once; Path.join is Go's Join. Spec/Defs.lean states Go's four rules on the elements of a path, as a rewriting relation, and Spec.lean proves that whatever order the rules are applied in, clean gives what they leave; that the result has no empty element or `.`, and `..` only at the start; that cleaning is idempotent; that join is what Go's documentation says; and that a clean relative path with no `..`, or any path cleaned as rooted, joined to a folder stays inside it. Go's own tests of Clean and Join are checked by the kernel. lake comparator --paranoid accepts all 10 statements under the six kernels. The differential tests compare 167,263 inputs with a model of the rules and with Go, Python and Node (0 failures, the differences named and confirmed by the model), and catch 22 of 22 planted bugs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AovXVgJBHGiN44TxQM4dcd --- .github/workflows/bar.yml | 21 + .gitignore | 1 + Main.lean | 104 +++++ Path.lean | 1 + Path/Basic.lean | 89 ++++ Proofs.lean | 5 + Proofs/Clean.lean | 274 ++++++++++++ Proofs/Join.lean | 128 ++++++ Proofs/Loop.lean | 461 +++++++++++++++++++ Proofs/Main.lean | 120 +++++ Proofs/Rules.lean | 332 ++++++++++++++ README.md | 94 +++- SPEC.md | 76 ++++ Spec.lean | 117 +++++ Spec/Defs.lean | 96 ++++ bar.toml | 33 ++ comparator.json | 10 + lake-manifest.json | 6 + lakefile.toml | 16 + lean-toolchain | 1 + tests/PLANTED-BUGS.md | 31 ++ tests/TEST-RESULTS.md | 39 ++ tests/oracles/go/go.mod | 3 + tests/oracles/go/main.go | 75 ++++ tests/oracles/node/oracle.js | 66 +++ tests/oracles/python/oracle.py | 49 ++ tests/plant_bugs.py | 182 ++++++++ tests/run_tests.py | 785 +++++++++++++++++++++++++++++++++ 28 files changed, 3193 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/bar.yml create mode 100644 Main.lean create mode 100644 Path.lean create mode 100644 Path/Basic.lean create mode 100644 Proofs.lean create mode 100644 Proofs/Clean.lean create mode 100644 Proofs/Join.lean create mode 100644 Proofs/Loop.lean create mode 100644 Proofs/Main.lean create mode 100644 Proofs/Rules.lean create mode 100644 SPEC.md create mode 100644 Spec.lean create mode 100644 Spec/Defs.lean create mode 100644 bar.toml create mode 100644 comparator.json create mode 100644 lake-manifest.json create mode 100644 lakefile.toml create mode 100644 lean-toolchain create mode 100644 tests/PLANTED-BUGS.md create mode 100644 tests/TEST-RESULTS.md create mode 100644 tests/oracles/go/go.mod create mode 100644 tests/oracles/go/main.go create mode 100644 tests/oracles/node/oracle.js create mode 100644 tests/oracles/python/oracle.py create mode 100644 tests/plant_bugs.py create mode 100644 tests/run_tests.py diff --git a/.github/workflows/bar.yml b/.github/workflows/bar.yml new file mode 100644 index 0000000..32457ed --- /dev/null +++ b/.github/workflows/bar.yml @@ -0,0 +1,21 @@ +name: verified bar + +on: + push: + pull_request: + # Rerun when the spec reviewer approves, or an approval is dismissed. + pull_request_review: + types: [submitted, dismissed] + +jobs: + bar: + # pull-requests: read lets the check confirm the spec approval on GitHub. + permissions: + contents: read + pull-requests: read + # Both lines pin the same commit of Proven-Commons/mission: bar v2, as + # proposed in its pull request #2. Move both to a release tag once + # mission publishes one. + uses: Proven-Commons/mission/.github/workflows/verified-bar.yml@6ba85c27abaaf07905c9316aa96dc29f7b5847a2 + with: + bar-ref: 6ba85c27abaaf07905c9316aa96dc29f7b5847a2 diff --git a/.gitignore b/.gitignore index 01f8cdb..ae73679 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .lake/ +__pycache__/ diff --git a/Main.lean b/Main.lean new file mode 100644 index 0000000..d9ab63e --- /dev/null +++ b/Main.lean @@ -0,0 +1,104 @@ +/- + Command line front end. + + path clean PATH + path join ELEM... + + print the path cleaned, or the elements joined and cleaned, then a + line end. `path join` with no elements, or only empty ones, prints an + empty line. Wrong arguments exit with 2. + + path serve + + answers requests on standard input, one per line, for the + differential tests. A path is sent as the hex digits of its UTF-8 + octets, two per octet, and the empty path is written `-`. + + clean HEX prints OK and the hex of the path cleaned + (clean) + join HEX... prints OK and the hex of the elements joined, + for zero or more elements (join) + + A malformed request prints `error`: an unknown operation, a `clean` + without exactly one path, hex digits that are not in pairs or not + lower case, or octets that are not UTF-8. A request line may end in + LF or CR LF. + + Every answer comes from the proved functions in `Path/`. The hex + coding and the request handling here are covered by the tests, not by + a proof. +-/ + +import Path + +def hexValue (c : Char) : Option Nat := + if '0' ≤ c ∧ c ≤ '9' then some (c.toNat - '0'.toNat) + else if 'a' ≤ c ∧ c ≤ 'f' then some (c.toNat - 'a'.toNat + 10) + else none + +/-- Octets from hex digits in pairs. Tail recursive, so that long + inputs do not use the stack. -/ +def hexBytes (cs : List Char) (acc : ByteArray) : Option ByteArray := + match cs with + | [] => some acc + | a :: b :: rest => + match hexValue a, hexValue b with + | some hi, some lo => hexBytes rest (acc.push (hi * 16 + lo).toUInt8) + | _, _ => none + | [_] => none + +/-- The path a field gives: `-` for the empty path, else the hex of + its UTF-8 octets. -/ +def fromHex (f : String) : Option String := + if f = "-" then some "" + else if f.isEmpty then none + else do + let bs ← hexBytes f.toList ByteArray.empty + String.fromUTF8? bs + +def hexDigit (n : Nat) : Char := + if n < 10 then Char.ofNat ('0'.toNat + n) else Char.ofNat ('a'.toNat + n - 10) + +/-- The field for a path: `-` when it is empty. -/ +def toHex (s : String) : String := + if s.isEmpty then "-" + else String.ofList (s.toUTF8.toList.flatMap fun b => + [hexDigit (b.toNat / 16), hexDigit (b.toNat % 16)]) + +/-- The answer to one request line, or `none` when it is malformed. -/ +def request (line : String) : Option String := + match line.splitOn " " with + | ["clean", h] => do + let p ← fromHex h + pure s!"OK {toHex (Path.clean p)}" + | "join" :: hs => do + let es ← hs.mapM fromHex + pure s!"OK {toHex (Path.join es)}" + | _ => none + +/-- The line without its line ending: LF, or CR LF. -/ +def chomp (line : String) : String := + let cs := line.toList + if cs.getLast? = some '\n' then + let cs := cs.dropLast + String.ofList (if cs.getLast? = some '\r' then cs.dropLast else cs) + else line + +def serve : IO Unit := do + let stdin ← IO.getStdin + let stdout ← IO.getStdout + repeat + let line ← stdin.getLine + if line.isEmpty then break + stdout.putStrLn ((request (chomp line)).getD "error") + stdout.flush + +def usage : String := + "usage: path clean PATH\n path join ELEM...\n path serve" + +def main (args : List String) : IO UInt32 := do + match args with + | ["clean", p] => IO.println (Path.clean p); return 0 + | "join" :: es => IO.println (Path.join es); return 0 + | ["serve"] => serve; return 0 + | _ => IO.eprintln usage; return 2 diff --git a/Path.lean b/Path.lean new file mode 100644 index 0000000..257fe8a --- /dev/null +++ b/Path.lean @@ -0,0 +1 @@ +import Path.Basic diff --git a/Path/Basic.lean b/Path/Basic.lean new file mode 100644 index 0000000..23b48d0 --- /dev/null +++ b/Path/Basic.lean @@ -0,0 +1,89 @@ +/- + Lexical path cleaning, as Go's `path.Clean` and `path.Join` do it + (go1.24.7, src/path/path.go). The loop below follows Go's line by + line. Go writes into a buffer and keeps its length `w`; here the + buffer is a list, kept reversed so that writing and removing at the + end take one step, and `w` is its length. Go reads the input by + index; here the loop takes the input apart from the front, so it + reads each character once. +-/ + +namespace Path + +/-- The part of Go's `..` case that backtracks. Go first removes one + character, then removes more while the output is longer than + `dotdot` and the character it last removed is not a slash. `out` + is the output, reversed, and `w` its length. -/ +def backtrack (dotdot : Nat) : List Char → Nat → List Char × Nat + | c :: out, w + 1 => if w > dotdot ∧ c ≠ '/' then backtrack dotdot out w else (out, w) + | out, w => (out, w) + +/-- Go's `..` case. If the output is longer than `dotdot`, it can + backtrack. Otherwise, in a path that is not rooted, it writes `..`, + after a slash when the output is not empty, and that is now where + `..` must stop. Gives the output, its length and the new `dotdot`. -/ +def parent (rooted : Bool) (dotdot : Nat) (out : List Char) (w : Nat) : + List Char × Nat × Nat := + if w > dotdot then + let (out, w) := backtrack dotdot out w + (out, w, dotdot) + else if !rooted then + let (out, w) := if w > 0 then ('/' :: out, w + 1) else (out, w) + ('.' :: '.' :: out, w + 2, w + 2) + else (out, w, dotdot) + +mutual +/-- Go's loop, at the start of a path element. -/ +def loop (rooted : Bool) (dotdot : Nat) (out : List Char) (w : Nat) : + List Char → List Char × Nat + | [] => (out, w) + -- empty path element + | '/' :: rest => loop rooted dotdot out w rest + -- . element + | ['.'] => (out, w) + | '.' :: rest@('/' :: _) => loop rooted dotdot out w rest + -- .. element: remove to last / + | ['.', '.'] => + let (out, w, _) := parent rooted dotdot out w + (out, w) + | '.' :: '.' :: rest@('/' :: _) => + let (out, w, dotdot) := parent rooted dotdot out w + loop rooted dotdot out w rest + -- real path element: add a slash if needed, then copy the element + | c :: rest => + let (out, w) := + if rooted && w != 1 || !rooted && w != 0 then ('/' :: out, w + 1) else (out, w) + copy rooted dotdot (c :: out) (w + 1) rest + +/-- The rest of a real path element, up to the next slash. Go stops + before the slash and its loop then skips it; here `copy` skips it. -/ +def copy (rooted : Bool) (dotdot : Nat) (out : List Char) (w : Nat) : + List Char → List Char × Nat + | [] => (out, w) + | '/' :: rest => loop rooted dotdot out w rest + | c :: rest => copy rooted dotdot (c :: out) (w + 1) rest +end + +/-- `Clean` on the characters of a path. -/ +def cleanChars (path : List Char) : List Char := + match path with + | [] => ['.'] + | c :: rest => + let rooted := c == '/' + let (out, w) := if rooted then loop true 1 ['/'] 1 rest else loop false 0 [] 0 path + -- Turn empty string into "." + if w == 0 then ['.'] else out.reverse + +/-- The shortest path equivalent to `path` by lexical processing alone, + as Go's `path.Clean` gives it. -/ +def clean (path : String) : String := + String.ofList (cleanChars path.toList) + +/-- Go's `path.Join`: the elements, the empty ones before the first + other one left out, separated by slashes, and cleaned. If there are + no elements, or all are empty, it gives the empty string. -/ +def join (elem : List String) : String := + if elem.all (·.isEmpty) then "" + else clean ("/".intercalate (elem.dropWhile (·.isEmpty))) + +end Path diff --git a/Proofs.lean b/Proofs.lean new file mode 100644 index 0000000..b9f0b73 --- /dev/null +++ b/Proofs.lean @@ -0,0 +1,5 @@ +import Proofs.Rules +import Proofs.Loop +import Proofs.Clean +import Proofs.Join +import Proofs.Main diff --git a/Proofs/Clean.lean b/Proofs/Clean.lean new file mode 100644 index 0000000..e7a3671 --- /dev/null +++ b/Proofs/Clean.lean @@ -0,0 +1,274 @@ +import Proofs.Loop + +/-! + `cleanChars` is the stack on the elements of its input, written out + as `render` writes elements. The facts about elements the statements + need: how to split a path, and what splitting paths joined by a slash + or written by `render` gives. +-/ + +namespace Spec.Proof + +open Spec Path + +theorem loop_elems (r : Bool) (es : List Element) (hne : es ≠ []) (hs : ∀ e ∈ es, '/' ∉ e) : + ∀ acc out w dd, Rel r acc out w dd → StackOk r acc → + ∃ out' w' dd', Rel r (es.foldl (push r) acc) out' w' dd' ∧ + loop r dd out w (List.intercalate ['/'] es) = (out', w') := by + induction es with + | nil => exact absurd rfl hne + | cons e es ih => + intro acc out w dd h hacc + obtain ⟨out1, w1, dd1, hrel, h1, h2⟩ := loop_elem r acc out w dd e (hs e (by simp)) h hacc + cases es with + | nil => exact ⟨out1, w1, dd1, hrel, by simpa using h1⟩ + | cons e' es => + rw [List.intercalate_cons_cons, List.append_assoc, List.singleton_append, h2, + List.foldl_cons] + exact ih (by simp) (fun e he => hs e (by simp [he])) _ _ _ _ hrel + (stackOk_push r acc e hacc) + +theorem rel_out (r : Bool) (acc : List Element) (out : List Char) (w dd : Nat) + (h : Rel r acc out w dd) : out.reverse = buf r acc.reverse ∧ w = out.length := by + obtain ⟨k, names, hacc, -, -, hout, hw, -⟩ := h + exact ⟨by rw [hacc, hout], hw⟩ + +theorem shape_ne_nil (r : Bool) (es : List Element) (h : Shape r es) : ∀ e ∈ es, e ≠ [] := by + obtain ⟨k, names, rfl, -, hnames⟩ := h + intro e he + rcases List.mem_append.mp he with he | he + · rw [(List.mem_replicate.mp he).2]; exact dotdot_ne_nil + · exact (hnames e he).1 + +theorem cleanChars_eq (p : List Char) (es : List Element) (h : Splits p es) : + cleanChars p = render (Rooted p) (nf (Rooted p) es) := by + obtain ⟨hne, hp, hs⟩ := h + have hnf := shape_ne_nil _ _ (nf_shape (Rooted p) es) + cases p with + | nil => + -- the one element is empty + have hes : es = [[]] := by + match es, hne, hp with + | [e], _, hp => simp at hp; rw [hp] + | e :: e' :: es, _, hp => simp at hp + subst hes + simp [cleanChars, Rooted, nf, push_nil, render, dot] + | cons c rest => + by_cases hc : c = '/' + · subst hc + -- the first element is the empty one before the root's slash + match es, hne, hp, hs with + | [e], _, hp, hs => + simp at hp + exact absurd (by rw [← hp]; simp) (hs e (by simp)) + | e :: e' :: es, _, hp, hs => + rw [List.intercalate_cons_cons, List.append_assoc, List.singleton_append] at hp + have he : e = [] := by + cases e with + | nil => rfl + | cons d e => + simp only [List.cons_append, List.cons.injEq] at hp + exact absurd (by rw [← hp.1]; simp) (hs (d :: e) (by simp)) + subst he + simp only [List.nil_append, List.cons.injEq, true_and] at hp + subst hp + obtain ⟨out', w', dd', hrel, hloop⟩ := loop_elems true (e' :: es) (by simp) + (fun x hx => hs x (by simp [hx])) [] ['/'] 1 1 (by simpa using rel_init true) + (stackOk_nil true) + obtain ⟨hout, hw⟩ := rel_out _ _ _ _ _ hrel + have hroot : Rooted ('/' :: List.intercalate ['/'] (e' :: es)) = true := by + simp [Rooted] + rw [hroot] + have hnfeq : nf true ([] :: e' :: es) = ((e' :: es).foldl (push true) []).reverse := by + simp [nf, push_nil] + have hw0 : (w' == 0) = false := by + have : out'.length > 0 := by + rw [← List.length_reverse, hout]; simp [buf] + simp; omega + simp only [cleanChars, beq_self_eq_true, ite_true, hloop, hw0, Bool.false_eq_true, + ite_false] + rw [hout, hnfeq] + simp [buf, render] + · have hroot : Rooted (c :: rest) = false := by simp [Rooted, hc] + rw [hroot] at hnf ⊢ + obtain ⟨out', w', dd', hrel, hloop⟩ := loop_elems false es hne hs [] [] 0 0 + (by simpa using rel_init false) (stackOk_nil false) + obtain ⟨hout, hw⟩ := rel_out _ _ _ _ _ hrel + have hc' : (c == '/') = false := by simp [hc] + simp only [cleanChars, hc', Bool.false_eq_true, ite_false] + rw [hp, hloop] + dsimp only + have hnfeq : nf false es = (es.foldl (push false) []).reverse := rfl + rw [hnfeq] at hnf ⊢ + by_cases hw0 : w' = 0 + · -- nothing is left + have hout0 : out' = [] := by + rw [hw] at hw0; exact List.length_eq_zero_iff.mp hw0 + rw [hout0] at hout + have hempty : (es.foldl (push false) []).reverse = [] := by + by_cases h0 : (es.foldl (push false) []).reverse = [] + · exact h0 + · exact absurd (by simpa [buf] using hout.symm) (intercalate_ne_nil _ h0 hnf) + simp [hw0, hempty, render, dot] + · have hw0' : (w' == 0) = false := by simp [hw0] + rw [hw0', ite_eq_right (by simp)] + have hne' : (es.foldl (push false) []).reverse ≠ [] := by + intro h0 + rw [h0] at hout + simp [buf] at hout + exact hw0 (by rw [hw, ← List.length_reverse, hout]; rfl) + rw [hout] + simp [buf, render, hne'] + +/-! ### Splitting -/ + +/-- The elements of a path. -/ +def elems : List Char → List Element + | [] => [[]] + | c :: cs => + if c = '/' then [] :: elems cs + else match elems cs with + | e :: es => (c :: e) :: es + | [] => [[c]] + +theorem elems_splits (p : List Char) : Splits p (elems p) := by + induction p with + | nil => exact ⟨by simp [elems], by simp [elems], by simp [elems]⟩ + | cons c cs ih => + obtain ⟨hne, hp, hs⟩ := ih + obtain ⟨e, es, hee⟩ := List.exists_cons_of_ne_nil hne + by_cases hc : c = '/' + · subst hc + refine ⟨by simp [elems], ?_, ?_⟩ + · simp only [elems, ite_true, hee] + rw [List.intercalate_cons_cons, ← hee, ← hp]; simp + · simp only [elems, ite_true, List.mem_cons] + rintro x (rfl | hx) + · simp + · exact hs x hx + · simp only [elems, hc, ite_false, hee] + refine ⟨by simp, ?_, ?_⟩ + · rw [List.intercalate_cons_cons_left, ← hee, ← hp] + · simp only [List.mem_cons] + rintro x (rfl | hx) + · simp only [List.mem_cons, not_or] + exact ⟨fun h => hc h.symm, hs e (by simp [hee])⟩ + · exact hs x (by simp [hee, hx]) + +theorem splits_append (x y : List Char) (E F : List Element) (hx : Splits x E) (hy : Splits y F) : + Splits (x ++ '/' :: y) (E ++ F) := by + obtain ⟨hE, rfl, hEs⟩ := hx + obtain ⟨hF, rfl, hFs⟩ := hy + refine ⟨by simp [hE], (intercalate_append E F hE hF).symm, ?_⟩ + intro e he + rcases List.mem_append.mp he with he | he + · exact hEs e he + · exact hFs e he + +theorem rooted_append (x y : List Char) (hx : x ≠ []) : Rooted (x ++ y) = Rooted x := by + obtain ⟨c, x, rfl⟩ := List.exists_cons_of_ne_nil hx + simp [Rooted] + +/-! ### What `render` writes -/ + +theorem push_foldl_nil (r : Bool) (acc es : List Element) : + (([] : Element) :: es).foldl (push r) acc = es.foldl (push r) acc := by + simp [push_nil] + +/-- `render` writes elements that split back into themselves, with an + empty element for the root and a `.` for an empty path, which the + stack drops. -/ +theorem splits_render (r : Bool) (E : List Element) (hE : ∀ e ∈ E, e ≠ [] ∧ '/' ∉ e) : + ∃ S, Splits (render r E) S ∧ ∀ r' acc, S.foldl (push r') acc = E.foldl (push r') acc := by + have hEs : ∀ e ∈ E, '/' ∉ e := fun e he => (hE e he).2 + cases r with + | true => + by_cases h0 : E = [] + · subst h0 + refine ⟨[[], []], ⟨by simp, by simp [render], by simp⟩, fun r' acc => by simp [push_nil]⟩ + · refine ⟨[] :: E, ⟨by simp, ?_, ?_⟩, fun r' acc => push_foldl_nil r' acc E⟩ + · obtain ⟨e, E', rfl⟩ := List.exists_cons_of_ne_nil h0 + simp [render] + · simp only [List.mem_cons] + rintro x (rfl | hx) + · simp + · exact hEs x hx + | false => + by_cases h0 : E = [] + · subst h0 + refine ⟨[dot], ⟨by simp, by simp [render, dot], by simp [dot]⟩, + fun r' acc => by simp [push_dot]⟩ + · refine ⟨E, ⟨h0, by simp [render, h0], hEs⟩, fun _ _ => rfl⟩ + +theorem rooted_render (r : Bool) (E : List Element) (hE : ∀ e ∈ E, e ≠ [] ∧ '/' ∉ e) : + Rooted (render r E) = r := by + cases r with + | true => simp [render, Rooted] + | false => + by_cases h0 : E = [] + · simp [render, h0, Rooted, dot] + · obtain ⟨e, E', rfl⟩ := List.exists_cons_of_ne_nil h0 + obtain ⟨he1, he2⟩ := hE e (by simp) + obtain ⟨c, e', rfl⟩ := List.exists_cons_of_ne_nil he1 + have hc : c ≠ '/' := fun h => he2 (by simp [h]) + cases E' with + | nil => simp [render, Rooted, hc] + | cons e2 E'' => simp [render, Rooted, hc] + +theorem normal_elems (r : Bool) (E : List Element) (hn : Normal r E) (hs : ∀ e ∈ E, '/' ∉ e) : + ∀ e ∈ E, e ≠ [] ∧ '/' ∉ e := + fun e he => ⟨shape_ne_nil r E (shape_of_normal r E hn) e he, hs e he⟩ + +/-- Cleaning what `render` writes gives it back. -/ +theorem cleanChars_render (r : Bool) (E : List Element) (hn : Normal r E) + (hs : ∀ e ∈ E, '/' ∉ e) : cleanChars (render r E) = render r E := by + have hE := normal_elems r E hn hs + obtain ⟨S, hS, hfold⟩ := splits_render r E hE + rw [cleanChars_eq _ _ hS, rooted_render r E hE] + unfold nf + rw [hfold r] + exact congrArg (render r) (nf_of_normal r E hn) + +/-! ### The stack keeps slash-free elements slash-free -/ + +theorem push_slashfree (r : Bool) (acc : List Element) (e : Element) + (hacc : ∀ x ∈ acc, '/' ∉ x) (he : '/' ∉ e) : ∀ x ∈ push r acc e, '/' ∉ x := by + intro x hx + unfold push at hx + split at hx + · exact hacc x hx + split at hx + · split at hx + · split at hx + · simp only [List.mem_cons] at hx + rcases hx with rfl | hx + · simp [dotdot] + · exact hacc x (List.mem_cons.mpr hx) + · exact hacc x (by simp [hx]) + · split at hx + · simp at hx + · simp only [List.mem_singleton] at hx; subst hx; simp [dotdot] + · simp only [List.mem_cons] at hx + rcases hx with rfl | hx + · exact he + · exact hacc x hx + +theorem nf_slashfree (r : Bool) (es : List Element) (hs : ∀ e ∈ es, '/' ∉ e) : + ∀ x ∈ nf r es, '/' ∉ x := by + have : ∀ acc, (∀ x ∈ acc, '/' ∉ x) → ∀ x ∈ es.foldl (push r) acc, '/' ∉ x := by + induction es with + | nil => intro acc h; exact h + | cons e es ih => + intro acc h + exact ih (fun x hx => hs x (by simp [hx])) _ + (push_slashfree r acc e h (hs e (by simp))) + intro x hx + exact this [] (by simp) x (List.mem_reverse.mp hx) + +/-- `clean p` and its elements. -/ +theorem clean_names' (p : String) : + Names (Path.clean p) (Rooted p.toList) (nf (Rooted p.toList) (elems p.toList)) := by + refine ⟨?_, nf_normal _ _, nf_slashfree _ _ (elems_splits _).2.2⟩ + rw [Path.clean, String.toList_ofList, cleanChars_eq _ _ (elems_splits _)] + +end Spec.Proof diff --git a/Proofs/Join.lean b/Proofs/Join.lean new file mode 100644 index 0000000..7490650 --- /dev/null +++ b/Proofs/Join.lean @@ -0,0 +1,128 @@ +import Proofs.Clean + +/-! + `join`: the elements of joined strings, and what cleaning them gives. +-/ + +namespace Spec.Proof + +open Spec Path + +theorem splits_intercalate (L : List (List Char)) (hL : L ≠ []) : + Splits (List.intercalate ['/'] L) (L.flatMap elems) := by + induction L with + | nil => exact absurd rfl hL + | cons x L ih => + cases L with + | nil => simpa using elems_splits x + | cons y L => + rw [List.intercalate_cons_cons, List.append_assoc, List.singleton_append, + List.flatMap_cons] + exact splits_append _ _ _ _ (elems_splits x) (ih (by simp)) + +theorem foldl_filter (r : Bool) (ss : List String) : + ∀ acc, ((ss.map String.toList).flatMap elems).foldl (push r) acc = + (((ss.filter (· ≠ "")).map String.toList).flatMap elems).foldl (push r) acc := by + induction ss with + | nil => intro acc; rfl + | cons s ss ih => + intro acc + by_cases hs : s = "" + · subst hs + simp only [List.map_cons, List.flatMap_cons, List.foldl_append, ne_eq, + not_true_eq_false, decide_false, Bool.false_eq_true, not_false_eq_true, + List.filter_cons_of_neg] + rw [show "".toList = [] by simp, show elems [] = [[]] from rfl] + simp only [List.foldl_cons, List.foldl_nil, push_nil] + exact ih acc + · rw [List.filter_cons_of_pos (by simpa using hs)] + simp only [List.map_cons, List.flatMap_cons, List.foldl_append] + exact ih _ + +theorem toList_slash_intercalate (ss : List String) : + ("/".intercalate ss).toList = List.intercalate ['/'] (ss.map String.toList) := by + rw [String.toList_intercalate]; rfl + +theorem rooted_intercalate (x : List Char) (L : List (List Char)) (hx : x ≠ []) : + Rooted (List.intercalate ['/'] (x :: L)) = Rooted x := by + cases L with + | nil => simp + | cons y L => rw [List.intercalate_cons_cons, List.append_assoc]; exact rooted_append _ _ hx + +/-- Leaving out the empty strings changes nothing once cleaned. -/ +theorem join_core (es : List String) (h : ∃ s ∈ es, s ≠ "") : + Path.clean ("/".intercalate (es.dropWhile (·.isEmpty))) = + Path.clean ("/".intercalate (es.filter (· ≠ ""))) := by + induction es with + | nil => obtain ⟨s, hs, -⟩ := h; simp at hs + | cons s es ih => + by_cases hs : s = "" + · subst hs + have h' : ∃ s ∈ es, s ≠ "" := by + obtain ⟨t, ht, htne⟩ := h + simp only [List.mem_cons] at ht + rcases ht with rfl | ht + · exact absurd rfl htne + · exact ⟨t, ht, htne⟩ + simpa using ih h' + · have hse : s.isEmpty = false := by simpa [String.isEmpty_iff] using hs + rw [List.dropWhile_cons, hse, List.filter_cons_of_pos (by simpa using hs)] + simp only [Bool.false_eq_true, ite_false] + apply String.toList_inj.mp + have hsl : s.toList ≠ [] := by simpa using hs + simp only [Path.clean, String.toList_ofList, toList_slash_intercalate, List.map_cons] + rw [cleanChars_eq _ _ (splits_intercalate _ (by simp)), + cleanChars_eq _ _ (splits_intercalate _ (by simp)), + rooted_intercalate _ _ hsl, rooted_intercalate _ _ hsl] + unfold nf + simp only [List.flatMap_cons, List.foldl_append] + rw [foldl_filter] + +theorem render_ne_nil (r : Bool) (E : List Element) (hE : ∀ e ∈ E, e ≠ [] ∧ '/' ∉ e) : + render r E ≠ [] := by + cases r with + | true => simp [render] + | false => + by_cases h0 : E = [] + · simp [render, h0, dot] + · simp only [render, Bool.false_eq_true, ite_false, h0] + exact intercalate_ne_nil E h0 (fun e he => (hE e he).1) + +/-- Joining a path `b` and the path `q` that `render` writes for names. -/ +theorem join_names (b q : String) (hb : b ≠ "") (r' : Bool) (eq : List Element) + (hq : q.toList = render r' eq) (hnames : ∀ e ∈ eq, IsName e ∧ '/' ∉ e) : + Names (Path.join [b, q]) (Rooted b.toList) (nf (Rooted b.toList) (elems b.toList) ++ eq) := by + have hE : ∀ e ∈ eq, e ≠ [] ∧ '/' ∉ e := fun e he => ⟨(hnames e he).1.1, (hnames e he).2⟩ + have hqne : q ≠ "" := by + intro h; subst h; exact render_ne_nil r' eq hE (by rw [← hq]; simp) + have hbl : b.toList ≠ [] := by simpa using hb + have hjoin : (Path.join [b, q]).toList = cleanChars (b.toList ++ '/' :: q.toList) := by + have hbe : b.isEmpty = false := by simpa [String.isEmpty_iff] using hb + have hqe : q.isEmpty = false := by simpa [String.isEmpty_iff] using hqne + simp only [Path.join, List.all_cons, hbe, hqe, Bool.false_and, List.all_nil, + Bool.false_eq_true, ite_false, List.dropWhile_cons] + simp [Path.clean] + obtain ⟨S, hS, hfold⟩ := splits_render r' eq hE + rw [← hq] at hS + have hsplit := splits_append _ _ _ _ (elems_splits b.toList) hS + have hnf : nf (Rooted b.toList) (elems b.toList ++ S) = + nf (Rooted b.toList) (elems b.toList) ++ eq := by + unfold nf + rw [List.foldl_append, hfold, foldl_names _ _ _ (fun e he => (hnames e he).1)] + simp + refine ⟨?_, ?_, ?_⟩ + · rw [hjoin, cleanChars_eq _ _ hsplit, rooted_append _ _ hbl, hnf] + · rw [← hnf]; exact nf_normal _ _ + · intro e he + rcases List.mem_append.mp he with he | he + · exact nf_slashfree _ _ (elems_splits _).2.2 e he + · exact (hnames e he).2 + +theorem names_of_shape_no_dotdot (r : Bool) (E : List Element) (hn : Normal r E) + (hd : dotdot ∉ E) : ∀ e ∈ E, IsName e := by + obtain ⟨k, names, rfl, -, hnames⟩ := shape_of_normal r E hn + cases k with + | zero => simpa using hnames + | succ k => exact absurd (by simp [List.replicate_succ]) hd + +end Spec.Proof diff --git a/Proofs/Loop.lean b/Proofs/Loop.lean new file mode 100644 index 0000000..1e4b095 --- /dev/null +++ b/Proofs/Loop.lean @@ -0,0 +1,461 @@ +import Path +import Proofs.Rules + +/-! + The code against the stack of `Proofs/Rules.lean`. Go's loop reads + the elements one by one, and after each its buffer holds the + elements on the stack, with a slash between each two and one in + front when the path is rooted. `dotdot` is the length of the `..` + elements at the start of the buffer, or 1 for the root. +-/ + +namespace Spec.Proof + +open Spec Path + +/-- The buffer for the elements `es`, first first. -/ +def buf (r : Bool) (es : List Element) : List Char := + if r then '/' :: List.intercalate ['/'] es else List.intercalate ['/'] es + +/-- The state of Go's loop holds the stack `acc`. -/ +def Rel (r : Bool) (acc : List Element) (out : List Char) (w dd : Nat) : Prop := + ∃ k names, acc.reverse = List.replicate k dotdot ++ names ∧ (r = true → k = 0) ∧ + (∀ e ∈ names, IsName e ∧ '/' ∉ e) ∧ + out.reverse = buf r (List.replicate k dotdot ++ names) ∧ w = out.length ∧ + dd = (buf r (List.replicate k dotdot)).length + +theorem rel_init (r : Bool) : + Rel r [] (if r then ['/'] else []) (if r then 1 else 0) (if r then 1 else 0) := by + refine ⟨0, [], by simp, fun _ => rfl, by simp, ?_, ?_, ?_⟩ <;> cases r <;> simp [buf] + +theorem intercalate_ne_nil (es : List Element) (h : es ≠ []) (hne : ∀ e ∈ es, e ≠ []) : + List.intercalate ['/'] es ≠ [] := by + cases es with + | nil => exact absurd rfl h + | cons e es => + have he := hne e (by simp) + cases es with + | nil => simpa using he + | cons e' es => simp + +theorem intercalate_snoc (es : List Element) (x : Element) (h : es ≠ []) : + List.intercalate ['/'] (es ++ [x]) = List.intercalate ['/'] es ++ '/' :: x := by + induction es with + | nil => exact absurd rfl h + | cons e es ih => + cases es with + | nil => simp + | cons e' es => + rw [List.cons_append, List.cons_append, List.intercalate_cons_cons, + List.intercalate_cons_cons, ← List.cons_append, ih (by simp)] + simp + +theorem buf_snoc (r : Bool) (es : List Element) (x : Element) (h : es ≠ []) : + buf r (es ++ [x]) = buf r es ++ '/' :: x := by + cases r <;> simp [buf, intercalate_snoc es x h] + +theorem buf_single (r : Bool) (x : Element) : + buf r [x] = (if r then ['/'] else []) ++ x := by + cases r <;> simp [buf] + +theorem buf_nil (r : Bool) : buf r [] = if r then ['/'] else [] := by + cases r <;> simp [buf] + +theorem intercalate_append (es fs : List Element) (h1 : es ≠ []) (h2 : fs ≠ []) : + List.intercalate ['/'] (es ++ fs) = + List.intercalate ['/'] es ++ '/' :: List.intercalate ['/'] fs := by + induction es with + | nil => exact absurd rfl h1 + | cons e es ih => + cases es with + | nil => + cases fs with + | nil => exact absurd rfl h2 + | cons f fs => simp + | cons e' es => + rw [List.cons_append, List.cons_append, List.intercalate_cons_cons, + List.intercalate_cons_cons, ← List.cons_append, ih (by simp)] + simp + +theorem buf_prefix_length (r : Bool) (es fs : List Element) : + (buf r es).length ≤ (buf r (es ++ fs)).length := by + by_cases h1 : es = [] + · subst h1; cases r <;> simp [buf] + by_cases h2 : fs = [] + · subst h2; simp + cases r <;> simp [buf, intercalate_append es fs h1 h2] + +/-! ### `backtrack` -/ + +theorem backtrack_sep (dd : Nat) (pre : List Char) (hpre : dd ≤ pre.length) : + ∀ xr : List Char, (∀ c ∈ xr, c ≠ '/') → + backtrack dd (xr ++ '/' :: pre) (pre.length + 1 + xr.length) = (pre, pre.length) := by + intro xr hxr + induction xr with + | nil => + simp only [List.nil_append, List.length_nil, Nat.add_zero] + rw [backtrack.eq_1] + simp + | cons c xr ih => + rw [List.cons_append, List.length_cons, show pre.length + 1 + (xr.length + 1) + = (pre.length + 1 + xr.length) + 1 by omega, backtrack.eq_1] + rw [ite_eq_left ⟨by omega, hxr c (by simp)⟩] + exact ih (fun c hc => hxr c (by simp [hc])) + +theorem backtrack_first (pre : List Char) : + ∀ xr : List Char, xr ≠ [] → (∀ c ∈ xr, c ≠ '/') → + backtrack pre.length (xr ++ pre) (pre.length + xr.length) = (pre, pre.length) := by + intro xr hne hxr + induction xr with + | nil => exact absurd rfl hne + | cons c xr ih => + rw [List.cons_append, List.length_cons, ← Nat.add_assoc, backtrack.eq_1] + cases xr with + | nil => simp + | cons c' xr => + rw [ite_eq_left ⟨by simp, hxr c (by simp)⟩] + exact ih (by simp) (fun c hc => hxr c (by simp [hc])) + +/-! ### One element -/ + +theorem names_snoc (k : Nat) (names : List Element) (acc : List Element) (x : Element) + (hx : x ≠ dotdot) (h : (x :: acc).reverse = List.replicate k dotdot ++ names) : + ∃ names', names = names' ++ [x] ∧ acc.reverse = List.replicate k dotdot ++ names' := by + cases hn : names.reverse with + | nil => + rw [List.reverse_eq_nil_iff] at hn + subst hn + rw [List.append_nil, List.reverse_cons] at h + cases k with + | zero => simp at h + | succ k => + rw [List.replicate_succ'] at h + exact absurd (List.append_inj_right' h rfl) (by simp [hx]) + | cons y rest => + have hnames : names = rest.reverse ++ [y] := by + rw [← List.reverse_reverse names, hn]; simp + subst hnames + rw [List.reverse_cons, ← List.append_assoc] at h + obtain ⟨h1, h2⟩ := List.append_inj' h rfl + simp only [List.cons.injEq, and_true] at h2 + subst h2 + exact ⟨rest.reverse, rfl, h1⟩ + +theorem rel_parent_name (r : Bool) (acc : List Element) (x : Element) (out : List Char) + (w dd : Nat) (hx : IsName x) (h : Rel r (x :: acc) out w dd) : + ∃ out' w', parent r dd out w = (out', w', dd) ∧ Rel r acc out' w' dd := by + obtain ⟨k, names, hacc, hk, hnames, hout, hw, hdd⟩ := h + obtain ⟨names', rfl, hacc'⟩ := names_snoc k names acc x hx.2.2 hacc + have hxs : '/' ∉ x := (hnames x (by simp)).2 + have hxr : ∀ c ∈ x.reverse, c ≠ '/' := fun c hc h => hxs (h ▸ List.mem_reverse.mp hc) + have hxne : x.reverse ≠ [] := by simpa using hx.1 + have hnames' : ∀ e ∈ names', IsName e ∧ '/' ∉ e := fun e he => hnames e (by simp [he]) + refine ⟨(buf r (List.replicate k dotdot ++ names')).reverse, + (buf r (List.replicate k dotdot ++ names')).length, ?_, + ⟨k, names', hacc', hk, hnames', by simp, by simp, hdd⟩⟩ + rw [← List.append_assoc] at hout + by_cases hL : List.replicate k dotdot ++ names' = [] + · -- `x` is the first element: the buffer is the root, or empty, and then `x` + have hk0 : k = 0 := by simpa using (List.append_eq_nil_iff.mp hL).1 + subst hk0 + rw [hL, List.nil_append, buf_single] at hout + rw [hL] + have hout' : out = x.reverse ++ (buf r []).reverse := by + rw [← List.reverse_reverse out, hout, buf_nil]; simp + have hdd' : dd = (buf r []).reverse.length := by rw [hdd]; simp + have hw' : w = (buf r []).reverse.length + x.reverse.length := by + rw [hw, hout']; simp; omega + unfold parent + rw [ite_eq_left (by rw [hw', hdd']; simp; exact List.length_pos_iff.mpr hx.1), hout', hw', hdd', + backtrack_first _ _ hxne hxr] + simp + · rw [buf_snoc r _ x hL] at hout + have hout' : out = x.reverse ++ '/' :: (buf r (List.replicate k dotdot ++ names')).reverse := by + rw [← List.reverse_reverse out, hout]; simp + have hle : dd ≤ (buf r (List.replicate k dotdot ++ names')).reverse.length := by + rw [hdd, List.length_reverse]; exact buf_prefix_length r _ _ + have hw' : w = (buf r (List.replicate k dotdot ++ names')).reverse.length + 1 + x.reverse.length := by + rw [hw, hout']; simp; omega + unfold parent + rw [ite_eq_left (by omega), hout', hw', backtrack_sep dd _ hle _ hxr] + simp + +/-- A path that is not rooted, with only `..` on the stack, keeps one more. -/ +theorem rel_parent_dotdots (k : Nat) (out : List Char) (w dd : Nat) + (h : Rel false (List.replicate k dotdot) out w dd) : + ∃ out' w', parent false dd out w = (out', w', w') ∧ + Rel false (dotdot :: List.replicate k dotdot) out' w' w' := by + obtain ⟨k', names, hacc, -, hnames, hout, hw, hdd⟩ := h + rw [List.reverse_replicate] at hacc + have hnames0 : names = [] := by + cases names with + | nil => rfl + | cons e names => + have hm : e ∈ List.replicate k dotdot := by rw [hacc]; simp + exact absurd (List.mem_replicate.mp hm).2 (hnames e (by simp)).1.2.2 + subst hnames0 + rw [List.append_nil] at hacc + have hk : k' = k := by simpa using (congrArg List.length hacc).symm + subst hk + rw [List.append_nil] at hout + have hwdd : w = dd := by rw [hw, hdd, ← List.length_reverse, hout] + have hpos : (w > 0) = (k' > 0) := by + rw [hw, ← List.length_reverse, hout] + cases k' with + | zero => simp [buf] + | succ k => + simp only [buf, Bool.false_eq_true, ite_false, gt_iff_lt, Nat.zero_lt_succ, iff_true, + eq_iff_iff] + exact List.length_pos_iff.mpr + (intercalate_ne_nil _ (by simp) (fun e he => by + rw [(List.mem_replicate.mp he).2]; exact dotdot_ne_nil)) + unfold parent + rw [ite_eq_right (by omega)] + simp only [Bool.not_false, ite_true] + by_cases hk0 : k' = 0 + · subst hk0 + have hw0 : ¬ w > 0 := by simp [hpos] + have hout0 : out = [] := by rw [← List.reverse_reverse out, hout]; simp [buf] + subst hout0 + simp only [hw0, ite_false] + refine ⟨_, _, rfl, 1, [], by simp, by simp, by simp, by simp [buf, dotdot], by simp [hw], + by simp [buf, dotdot, hw]⟩ + · have hw0 : w > 0 := by rw [hpos]; omega + simp only [hw0, ite_true] + have hbuf : ('.' :: '.' :: '/' :: out).reverse = buf false (List.replicate (k' + 1) dotdot) := by + rw [List.replicate_succ', buf_snoc false _ _ (by simpa using hk0)] + simp [hout, dotdot] + refine ⟨_, _, rfl, k' + 1, [], by simp [List.replicate_succ'], by simp, by simp, + by simpa using hbuf, by simp [hw], ?_⟩ + rw [← hbuf]; simp [hw] + +/-- The copy of a name, up to the next slash. -/ +theorem copy_name (r : Bool) (dd : Nat) (cs : List Char) (hcs : '/' ∉ cs) : + ∀ out w, copy r dd out w cs = (cs.reverse ++ out, w + cs.length) ∧ + ∀ rest, copy r dd out w (cs ++ '/' :: rest) = + loop r dd (cs.reverse ++ out) (w + cs.length) rest := by + induction cs with + | nil => + intro out w + refine ⟨by rw [copy.eq_1]; simp, fun rest => ?_⟩ + rw [List.nil_append, copy.eq_2]; simp + | cons c cs ih => + intro out w + have hc : c ≠ '/' := fun h => hcs (by simp [h]) + have hcs' : '/' ∉ cs := fun h => hcs (by simp [h]) + obtain ⟨ih1, ih2⟩ := ih hcs' (c :: out) (w + 1) + refine ⟨?_, fun rest => ?_⟩ + · rw [copy.eq_3 _ _ _ _ _ _ hc, ih1]; simp; omega + · rw [List.cons_append, copy.eq_3 _ _ _ _ _ _ hc, ih2 rest] + simp only [List.reverse_cons, List.append_assoc, List.singleton_append, List.length_cons] + rw [show w + 1 + cs.length = w + (cs.length + 1) by omega] + +/-- `loop` on a name starts its copy. -/ +theorem loop_name (r : Bool) (dd : Nat) (out : List Char) (w : Nat) (c : Char) (cs t : List Char) + (hn : IsName (c :: cs)) (hs : '/' ∉ c :: cs) (ht : t = [] ∨ ∃ rest, t = '/' :: rest) : + loop r dd out w (c :: (cs ++ t)) = + (let (out, w) := + if (r && w != 1 || !r && w != 0) = true then ('/' :: out, w + 1) else (out, w) + copy r dd (c :: out) (w + 1) (cs ++ t)) := by + obtain ⟨hn1, hn2, hn3⟩ := hn + have hc : c ≠ '/' := fun h => hs (by simp [h]) + rw [loop.eq_7] + · exact fun h => hc h + · intro h1 h2 + subst h1 + rcases List.append_eq_nil_iff.mp h2 with ⟨rfl, -⟩ + exact hn2 rfl + · intro tail h1 h2 + subst h1 + cases cs with + | nil => + exact hn2 rfl + | cons c2 cs => + simp only [List.cons_append, List.cons.injEq] at h2 + exact hs (by simp [h2.1]) + · intro h1 h2 + subst h1 + cases cs with + | nil => + simp only [List.nil_append] at h2 + rcases ht with rfl | ⟨rest, rfl⟩ <;> simp at h2 + | cons c2 cs => + simp only [List.cons_append, List.cons.injEq, List.append_eq_nil_iff] at h2 + obtain ⟨rfl, rfl, -⟩ := h2 + exact hn3 rfl + · intro tail h1 h2 + subst h1 + cases cs with + | nil => + simp only [List.nil_append] at h2 + rcases ht with rfl | ⟨rest, rfl⟩ <;> simp at h2 + | cons c2 cs => + simp only [List.cons_append, List.cons.injEq] at h2 + obtain ⟨rfl, h2⟩ := h2 + cases cs with + | nil => + exact hn3 rfl + | cons c3 cs => + simp only [List.cons_append, List.cons.injEq] at h2 + exact hs (by simp [h2.1]) + +theorem rel_name (r : Bool) (acc : List Element) (out : List Char) (w dd : Nat) + (e : Element) (hn : IsName e) (hs : '/' ∉ e) (h : Rel r acc out w dd) + (out1 : List Char) (w1 : Nat) + (hc : (if (r && w != 1 || !r && w != 0) = true then ('/' :: out, w + 1) else (out, w)) + = (out1, w1)) : + Rel r (e :: acc) (e.reverse ++ out1) (w1 + e.length) dd := by + obtain ⟨k, names, hacc, hk, hnames, hout, hw, hdd⟩ := h + have hw1 : w1 = out1.length := by + split at hc <;> simp only [Prod.mk.injEq] at hc <;> obtain ⟨rfl, rfl⟩ := hc <;> simp [hw] + refine ⟨k, names ++ [e], by simp [hacc], hk, ?_, ?_, by simp [hw1, Nat.add_comm], hdd⟩ + · intro x hx + simp only [List.mem_append, List.mem_singleton] at hx + rcases hx with hx | rfl + · exact hnames x hx + · exact ⟨hn, hs⟩ + · rw [← List.append_assoc] + by_cases hL : List.replicate k dotdot ++ names = [] + · rw [hL, List.nil_append, buf_single] + rw [hL, buf_nil] at hout + have hw' : w = if r then 1 else 0 := by + rw [hw, ← List.length_reverse, hout]; cases r <;> simp + have : out1 = out := by + cases r <;> simp [hw'] at hc <;> exact hc.1.symm + rw [this]; simp [hout] + · rw [buf_snoc r _ e hL] + have hne : ∀ x ∈ List.replicate k dotdot ++ names, x ≠ [] := by + intro x hx + rcases List.mem_append.mp hx with hx | hx + · rw [(List.mem_replicate.mp hx).2]; exact dotdot_ne_nil + · exact (hnames x hx).1.1 + have hI := List.length_pos_iff.mpr (intercalate_ne_nil _ hL hne) + have hw' : w = (buf r (List.replicate k dotdot ++ names)).length := by + rw [hw, ← List.length_reverse, hout] + have : out1 = '/' :: out := by + have hcond : (r && w != 1 || !r && w != 0) = true := by + cases r <;> simp [buf] at hw' ⊢ <;> omega + rw [ite_eq_left hcond] at hc + simp only [Prod.mk.injEq] at hc + exact hc.1.symm + rw [this]; simp [hout] + +/-- The top of the stack is `..` only when the path is not rooted and + the whole stack is `..`. -/ +theorem rel_top_dotdot (r : Bool) (acc : List Element) (out : List Char) (w dd : Nat) + (h : Rel r (dotdot :: acc) out w dd) : + r = false ∧ ∃ k, dotdot :: acc = List.replicate k dotdot := by + obtain ⟨k, names, hacc, hk, hnames, -⟩ := h + have hn : names = [] := by + cases hrev : names.reverse with + | nil => simpa using hrev + | cons y rest => + have hnames' : names = rest.reverse ++ [y] := by + rw [← List.reverse_reverse names, hrev]; simp + subst hnames' + rw [List.reverse_cons, ← List.append_assoc] at hacc + obtain ⟨-, h2⟩ := List.append_inj' hacc rfl + simp only [List.cons.injEq, and_true] at h2 + subst h2 + exact absurd rfl (hnames dotdot (by simp)).1.2.2 + subst hn + rw [List.append_nil] at hacc + have hacc2 : dotdot :: acc = List.replicate k dotdot := by + rw [← List.reverse_reverse (dotdot :: acc), hacc, List.reverse_replicate] + refine ⟨?_, k, hacc2⟩ + cases r with + | false => rfl + | true => + have := hk rfl + subst this + simp at hacc2 + +/-- Go's loop on one element, at the end or followed by a slash. -/ +theorem loop_elem (r : Bool) (acc : List Element) (out : List Char) (w dd : Nat) + (e : Element) (hs : '/' ∉ e) (h : Rel r acc out w dd) (hacc : StackOk r acc) : + ∃ out' w' dd', Rel r (push r acc e) out' w' dd' ∧ + loop r dd out w e = (out', w') ∧ + ∀ rest, loop r dd out w (e ++ '/' :: rest) = loop r dd' out' w' rest := by + by_cases h1 : e = [] + · subst h1 + refine ⟨out, w, dd, by rw [push_nil]; exact h, by rw [loop.eq_1], fun rest => ?_⟩ + rw [List.nil_append, loop.eq_2] + by_cases h2 : e = dot + · subst h2 + refine ⟨out, w, dd, by rw [push_dot]; exact h, by simp [dot, loop.eq_3], fun rest => ?_⟩ + simp only [dot, List.cons_append, List.nil_append] + rw [loop.eq_4, loop.eq_2] + by_cases h3 : e = dotdot + · subst h3 + -- what `parent` does, with the stack it leaves + have hp : ∃ out' w' dd', parent r dd out w = (out', w', dd') ∧ + Rel r (push r acc dotdot) out' w' dd' := by + cases acc with + | nil => + rw [push_dotdot_nil] + cases r with + | false => + obtain ⟨out', w', hp, hrel⟩ := rel_parent_dotdots 0 out w dd (by simpa using h) + exact ⟨out', w', w', hp, by simpa using hrel⟩ + | true => + obtain ⟨k, names, hacc', hk, -, hout, hw, hdd⟩ := h + have hk0 := hk rfl + subst hk0 + have hn0 : names = [] := by simpa using hacc'.symm + subst hn0 + refine ⟨out, w, dd, ?_, ⟨0, [], by simp, fun _ => rfl, by simp, hout, hw, hdd⟩⟩ + have : w = dd := by rw [hw, hdd, ← List.length_reverse, hout]; simp + unfold parent + simp [this] + | cons x acc => + rw [push_dotdot_cons] + by_cases hx : x = dotdot + · subst hx + simp only [ite_true] + -- the whole stack is `..` + obtain ⟨hr, k, hk⟩ := rel_top_dotdot r acc out w dd h + subst hr + rw [hk] at h ⊢ + obtain ⟨out', w', hp, hrel⟩ := rel_parent_dotdots k out w dd h + exact ⟨out', w', w', hp, hrel⟩ + · simp only [hx, ite_false] + have hxn : IsName x := by + obtain ⟨k, names, hacc', -, hnames, -⟩ := h + have hm : x ∈ List.replicate k dotdot ++ names := by rw [← hacc']; simp + rcases List.mem_append.mp hm with hm | hm + · exact absurd (List.mem_replicate.mp hm).2 hx + · exact (hnames x hm).1 + obtain ⟨out', w', hp, hrel⟩ := rel_parent_name r acc x out w dd hxn h + exact ⟨out', w', dd, hp, hrel⟩ + obtain ⟨out', w', dd', hp, hrel⟩ := hp + refine ⟨out', w', dd', hrel, ?_, fun rest => ?_⟩ + · simp only [dotdot]; rw [loop.eq_5, hp] + · simp only [dotdot, List.cons_append, List.nil_append] + rw [loop.eq_6, hp] + simp only + rw [loop.eq_2] + · -- a name + have hn : IsName e := ⟨h1, h2, h3⟩ + obtain ⟨c, cs, rfl⟩ := List.exists_cons_of_ne_nil h1 + have hcs : '/' ∉ cs := fun h => hs (by simp [h]) + obtain ⟨out1, w1, hc⟩ : ∃ out1 w1, + (if (r && w != 1 || !r && w != 0) = true then ('/' :: out, w + 1) else (out, w)) + = (out1, w1) := ⟨_, _, rfl⟩ + have hrel := rel_name r acc out w dd (c :: cs) hn hs h out1 w1 hc + refine ⟨(c :: cs).reverse ++ out1, w1 + (c :: cs).length, dd, + by rw [push_name r acc _ hn]; exact hrel, ?_, fun rest => ?_⟩ + · have := loop_name r dd out w c cs [] hn hs (Or.inl rfl) + rw [List.append_nil] at this + rw [this, hc] + dsimp only + rw [(copy_name r dd cs hcs _ _).1] + simp only [List.reverse_cons, List.append_assoc, List.singleton_append, List.length_cons] + rw [show w1 + 1 + cs.length = w1 + (cs.length + 1) by omega] + · have := loop_name r dd out w c cs ('/' :: rest) hn hs (Or.inr ⟨rest, rfl⟩) + rw [List.cons_append, this, hc] + dsimp only + rw [(copy_name r dd cs hcs _ _).2 rest] + simp only [List.reverse_cons, List.append_assoc, List.singleton_append, List.length_cons] + rw [show w1 + 1 + cs.length = w1 + (cs.length + 1) by omega] + +end Spec.Proof diff --git a/Proofs/Main.lean b/Proofs/Main.lean new file mode 100644 index 0000000..7dee5db --- /dev/null +++ b/Proofs/Main.lean @@ -0,0 +1,120 @@ +/- + The statements of `Spec.lean`, each proved from the lemmas in the + other files of `Proofs/`. `lake comparator` checks that each theorem + here is exactly the one in `Spec.lean`. +-/ + +import Proofs.Join + +namespace Spec + +open Spec.Proof + +theorem clean_rules (p : String) (es es' : List Element) + (hp : Splits p.toList es) (hs : Steps (Rooted p.toList) es es') + (hn : Normal (Rooted p.toList) es') : + (Path.clean p).toList = render (Rooted p.toList) es' := by + rw [Path.clean, String.toList_ofList, cleanChars_eq _ _ hp, ← steps_normal_eq _ _ _ hs hn] + +theorem rules_end (p : String) : + ∃ es es', Splits p.toList es ∧ Steps (Rooted p.toList) es es' ∧ + Normal (Rooted p.toList) es' := + ⟨elems p.toList, nf _ (elems p.toList), elems_splits _, steps_nf_self _ _, nf_normal _ _⟩ + +theorem normal_iff (rooted : Bool) (es : List Element) : + Normal rooted es ↔ + ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ + ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := + normal_iff_shape rooted es + +theorem clean_names (p : String) : ∃ es, Names (Path.clean p) (Rooted p.toList) es := + ⟨_, clean_names' p⟩ + +theorem clean_idem (p : String) : Path.clean (Path.clean p) = Path.clean p := by + obtain ⟨h1, h2, h3⟩ := clean_names' p + apply String.toList_inj.mp + show (String.ofList (Path.cleanChars (Path.clean p).toList)).toList = _ + rw [String.toList_ofList, h1, cleanChars_render _ _ h2 h3] + +theorem join_eq (es : List String) : + Path.join es = + if es.all (· = "") then "" else Path.clean ("/".intercalate (es.filter (· ≠ ""))) := by + have hall : es.all (·.isEmpty) = es.all (· = "") := by + congr 1; funext s; by_cases h : s = "" <;> simp [h, String.isEmpty_iff] + unfold Path.join + rw [hall] + split + · rfl + · rename_i h + exact join_core es (by simpa using h) + +theorem join_inside (b q : String) (eq : List Element) (hq : Names q false eq) + (hd : dotdot ∉ eq) : + ∃ eb, Names (Path.clean b) (Rooted b.toList) eb ∧ + Names (Path.join [b, q]) (Rooted b.toList) (eb ++ eq) := by + obtain ⟨hq1, hq2, hq3⟩ := hq + have hnames := names_of_shape_no_dotdot false eq hq2 hd + refine ⟨_, clean_names' b, ?_⟩ + by_cases hb : b = "" + · subst hb + have hE : ∀ e ∈ eq, e ≠ [] ∧ '/' ∉ e := fun e he => ⟨(hnames e he).1, hq3 e he⟩ + have hqne : q ≠ "" := by + intro h; subst h; exact render_ne_nil false eq hE (by rw [← hq1]; simp) + have hqe : q.isEmpty = false := by simpa [String.isEmpty_iff] using hqne + have hjoin : Path.join ["", q] = Path.clean q := by + simp [Path.join, hqe] + have hn0 : nf (Rooted "".toList) (elems "".toList) = [] := by + simp [Rooted, elems, nf, push_nil] + rw [hjoin, hn0, List.nil_append] + simp only [show Rooted "".toList = false by simp [Rooted]] + refine ⟨?_, hq2, hq3⟩ + rw [Path.clean, String.toList_ofList, hq1, cleanChars_render _ _ hq2 hq3] + · exact join_names b q hb false eq hq1 (fun e he => ⟨hnames e he, hq3 e he⟩) + +theorem join_root_inside (b u : String) (hb : b ≠ "") : + ∃ eb ext, dotdot ∉ ext ∧ Names (Path.clean b) (Rooted b.toList) eb ∧ + Names (Path.join [b, Path.clean ("/" ++ u)]) (Rooted b.toList) (eb ++ ext) := by + obtain ⟨h1, h2, h3⟩ := clean_names' ("/" ++ u) + have hr : Rooted ("/" ++ u).toList = true := by simp [Rooted] + rw [hr] at h1 h2 h3 + have hnames : ∀ e ∈ nf true (elems ("/" ++ u).toList), IsName e := by + obtain ⟨k, names, hE, hk, hn⟩ := shape_of_normal _ _ h2 + rw [hE, hk rfl]; simpa using hn + refine ⟨_, _, fun h => (hnames _ h).2.2 rfl, clean_names' b, + join_names b _ hb true _ h1 (fun e he => ⟨hnames e he, h3 e he⟩)⟩ + +theorem go_clean_tests : + ∀ t ∈ [ + -- Already clean + ("", "."), ("abc", "abc"), ("abc/def", "abc/def"), ("a/b/c", "a/b/c"), + (".", "."), ("..", ".."), ("../..", "../.."), ("../../abc", "../../abc"), + ("/abc", "/abc"), ("/", "/"), + -- Remove trailing slash + ("abc/", "abc"), ("abc/def/", "abc/def"), ("a/b/c/", "a/b/c"), ("./", "."), + ("../", ".."), ("../../", "../.."), ("/abc/", "/abc"), + -- Remove doubled slash + ("abc//def//ghi", "abc/def/ghi"), ("//abc", "/abc"), ("///abc", "/abc"), + ("//abc//", "/abc"), ("abc//", "abc"), + -- Remove . elements + ("abc/./def", "abc/def"), ("/./abc/def", "/abc/def"), ("abc/.", "abc"), + -- Remove .. elements + ("abc/def/ghi/../jkl", "abc/def/jkl"), ("abc/def/../ghi/../jkl", "abc/jkl"), + ("abc/def/..", "abc"), ("abc/def/../..", "."), ("/abc/def/../..", "/"), + ("abc/def/../../..", ".."), ("/abc/def/../../..", "/"), + ("abc/def/../../../ghi/jkl/../../../mno", "../../mno"), + -- Combinations + ("abc/./../def", "def"), ("abc//./../def", "def"), + ("abc/../../././../def", "../../def")], + Path.clean t.1 = t.2 ∧ Path.clean t.2 = t.2 := by + decide +kernel + +theorem go_join_tests : + ∀ t ∈ [ + ([], ""), ([""], ""), (["a"], "a"), + (["a", "b"], "a/b"), (["a", ""], "a"), (["", "b"], "b"), (["/", "a"], "/a"), + (["/", ""], "/"), (["a/", "b"], "a/b"), (["a/", ""], "a"), (["", ""], ""), + (["", "/etc"], "/etc")], + Path.join t.1 = t.2 := by + decide +kernel + +end Spec diff --git a/Proofs/Rules.lean b/Proofs/Rules.lean new file mode 100644 index 0000000..f316c31 --- /dev/null +++ b/Proofs/Rules.lean @@ -0,0 +1,332 @@ +import Spec.Defs + +/-! + The rules on elements. `nf` applies them with a stack, left to + right; every rule leaves `nf` unchanged, the rules lead from any + elements to `nf` of them, and `nf` of elements to which no rule + applies is those elements. So the rules end in one place, whatever + their order. +-/ + +namespace Spec.Proof + +open Spec + +/-- A name: an element that is not empty, `.` or `..`. -/ +def IsName (e : Element) : Prop := e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot + +/-- One element of the stack. `acc` is the elements kept, last first. -/ +def push (rooted : Bool) (acc : List Element) (e : Element) : List Element := + if e = [] ∨ e = dot then acc + else if e = dotdot then + match acc with + | x :: acc' => if x = dotdot then dotdot :: acc else acc' + | [] => if rooted then [] else [dotdot] + else e :: acc + +/-- The elements left by the stack. -/ +def nf (rooted : Bool) (es : List Element) : List Element := + (es.foldl (push rooted) []).reverse + +theorem dot_ne_nil : dot ≠ [] := by simp [dot] +theorem dotdot_ne_nil : dotdot ≠ [] := by simp [dotdot] +theorem dotdot_ne_dot : dotdot ≠ dot := by simp [dot, dotdot] + +theorem push_nil (r : Bool) (acc : List Element) : push r acc [] = acc := by + simp [push] + +theorem push_dot (r : Bool) (acc : List Element) : push r acc dot = acc := by + simp [push] + +theorem push_name (r : Bool) (acc : List Element) (e : Element) (h : IsName e) : + push r acc e = e :: acc := by + obtain ⟨h1, h2, h3⟩ := h + simp [push, h1, h2, h3] + +theorem push_dotdot_cons (r : Bool) (x : Element) (acc : List Element) : + push r (x :: acc) dotdot = if x = dotdot then dotdot :: x :: acc else acc := by + simp [push, dotdot_ne_nil, dotdot_ne_dot] + +theorem push_dotdot_nil (r : Bool) : + push r [] dotdot = if r then [] else [dotdot] := by + simp [push, dotdot_ne_nil, dotdot_ne_dot] + +theorem foldl_names (r : Bool) (acc names : List Element) (h : ∀ e ∈ names, IsName e) : + names.foldl (push r) acc = names.reverse ++ acc := by + induction names generalizing acc with + | nil => rfl + | cons e names ih => + rw [List.foldl_cons, push_name r acc e (h e (by simp)), + ih _ (fun e he => h e (by simp [he]))] + simp + +theorem foldl_dotdots (acc : List Element) (k : Nat) (hacc : ∀ x ∈ acc, x = dotdot) : + (List.replicate k dotdot).foldl (push false) acc = List.replicate k dotdot ++ acc := by + induction k generalizing acc with + | zero => rfl + | succ k ih => + rw [List.replicate_succ, List.foldl_cons] + have : push false acc dotdot = dotdot :: acc := by + cases acc with + | nil => simp [push_dotdot_nil] + | cons x acc => simp [push_dotdot_cons, hacc x (by simp)] + rw [this, ih _ (by + intro x hx + simp only [List.mem_cons] at hx + rcases hx with hx | hx + · exact hx + · exact hacc x hx)] + rw [show List.replicate k dotdot ++ dotdot :: acc + = (List.replicate k dotdot ++ [dotdot]) ++ acc by simp, + ← List.replicate_succ', List.replicate_succ, List.cons_append] + +/-! ### Each rule leaves `nf` unchanged -/ + +theorem step_foldl (r : Bool) (es es' : List Element) (h : Step r es es') : + es.foldl (push r) [] = es'.foldl (push r) [] := by + cases h with + | empty xs ys => simp [List.foldl_append, push_nil] + | dot xs ys => simp [List.foldl_append, push_dot] + | parent xs e ys h1 h2 h3 => + simp only [List.foldl_append, List.foldl_cons] + rw [push_name r _ e ⟨h1, h2, h3⟩, push_dotdot_cons] + simp [h3] + | root ys hr => + simp [List.foldl_cons, push_dotdot_nil, hr] + +theorem steps_nf (r : Bool) (es es' : List Element) (h : Steps r es es') : + nf r es = nf r es' := by + induction h with + | refl => rfl + | step h _ ih => rw [← ih]; unfold nf; rw [step_foldl r _ _ h] + +/-! ### The shape of elements to which no rule applies -/ + +/-- Some `..`, none if rooted, then names. -/ +def Shape (rooted : Bool) (es : List Element) : Prop := + ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ + ∀ e ∈ names, IsName e + +theorem normal_tail (r : Bool) (e : Element) (es : List Element) (h : Normal r (e :: es)) : + Normal false es := by + intro es' hs + cases hs with + | empty xs ys => exact h _ (Step.empty (e :: xs) ys) + | dot xs ys => exact h _ (Step.dot (e :: xs) ys) + | parent xs x ys h1 h2 h3 => exact h _ (Step.parent (e :: xs) x ys h1 h2 h3) + | root ys hr => cases hr + +theorem shape_of_normal (r : Bool) (es : List Element) (h : Normal r es) : Shape r es := by + induction es generalizing r with + | nil => exact ⟨0, [], rfl, fun _ => rfl, by simp⟩ + | cons e es ih => + obtain ⟨k, names, hes, -, hnames⟩ := ih false (normal_tail r e es h) + have he1 : e ≠ [] := by + rintro rfl; exact h _ (Step.empty [] es) + have he2 : e ≠ dot := by + rintro rfl; exact h _ (Step.dot [] es) + by_cases he3 : e = dotdot + · subst he3 + cases r with + | true => exact absurd (Step.root es rfl) (h _) + | false => + exact ⟨k + 1, names, by simp [hes, List.replicate_succ], by simp, hnames⟩ + · refine ⟨0, e :: names, ?_, fun _ => rfl, ?_⟩ + · cases k with + | zero => simpa using hes + | succ k => + exfalso + rw [List.replicate_succ, List.cons_append] at hes + subst hes + exact h _ (Step.parent [] e _ he1 he2 he3) + · intro x hx + simp only [List.mem_cons] at hx + rcases hx with rfl | hx + · exact ⟨he1, he2, he3⟩ + · exact hnames x hx + +theorem normal_of_shape (r : Bool) (es : List Element) (h : Shape r es) : Normal r es := by + obtain ⟨k, names, hes, hk, hnames⟩ := h + intro es' hs + have hdd : dotdot ∉ names := fun hm => (hnames _ hm).2.2 rfl + have hrep : ∀ x ∈ List.replicate k dotdot, x = dotdot := fun x hx => + (List.mem_replicate.mp hx).2 + cases hs with + | empty xs ys => + have : ([] : Element) ∈ List.replicate k dotdot ++ names := by + rw [← hes]; simp + rcases List.mem_append.mp this with hm | hm + · exact dotdot_ne_nil (hrep _ hm).symm + · exact (hnames _ hm).1 rfl + | dot xs ys => + have : dot ∈ List.replicate k dotdot ++ names := by + rw [← hes]; simp + rcases List.mem_append.mp this with hm | hm + · exact dotdot_ne_dot (hrep _ hm).symm + · exact (hnames _ hm).2.1 rfl + | parent xs e ys h1 h2 h3 => + rcases List.append_eq_append_iff.mp hes with ⟨c, hc1, hc2⟩ | ⟨c, hc1, hc2⟩ + · cases c with + | nil => + simp only [List.nil_append] at hc2 + exact hdd (by rw [← hc2]; simp) + | cons x c => + simp only [List.cons_append, List.cons.injEq] at hc2 + obtain ⟨rfl, -⟩ := hc2 + exact h3 (hrep e (by rw [hc1]; simp)) + · exact hdd (by rw [hc2]; simp) + | root ys hr => + have : dotdot ∈ List.replicate k dotdot ++ names := by rw [← hes]; simp + rw [hk hr] at this + exact hdd (by simpa using this) + +theorem normal_iff_shape (r : Bool) (es : List Element) : Normal r es ↔ Shape r es := + ⟨shape_of_normal r es, normal_of_shape r es⟩ + +/-! ### The stack: what it holds, and the rules lead to it -/ + +theorem Steps.trans {r : Bool} {a b c : List Element} (h1 : Steps r a b) (h2 : Steps r b c) : + Steps r a c := by + induction h1 with + | refl => exact h2 + | step h _ ih => exact Steps.step h (ih h2) + +theorem Steps.single {r : Bool} {a b : List Element} (h : Step r a b) : Steps r a b := + Steps.step h (Steps.refl _) + +/-- The stack, first element last, is in shape. -/ +def StackOk (r : Bool) (acc : List Element) : Prop := Shape r acc.reverse + +theorem stackOk_nil (r : Bool) : StackOk r [] := ⟨0, [], rfl, fun _ => rfl, by simp⟩ + +theorem stackOk_push (r : Bool) (acc : List Element) (e : Element) (h : StackOk r acc) : + StackOk r (push r acc e) := by + obtain ⟨k, names, hacc, hk, hnames⟩ := h + by_cases h1 : e = [] ∨ e = dot + · rcases h1 with rfl | rfl + · rw [push_nil]; exact ⟨k, names, hacc, hk, hnames⟩ + · rw [push_dot]; exact ⟨k, names, hacc, hk, hnames⟩ + have hne : e ≠ [] ∧ e ≠ dot := by + constructor <;> intro h <;> exact h1 (by simp [h]) + by_cases h2 : e = dotdot + · subst h2 + -- the stack, last element first, is the names reversed, then the `..` + have hacc' : acc = names.reverse ++ List.replicate k dotdot := by + rw [← List.reverse_reverse acc, hacc]; simp + cases hrev : names.reverse with + | nil => + rw [List.reverse_eq_nil_iff] at hrev + subst hrev + cases k with + | zero => + simp at hacc' + subst hacc' + rw [push_dotdot_nil] + cases r + · exact ⟨1, [], by simp, by simp, by simp⟩ + · exact ⟨0, [], by simp, by simp, by simp⟩ + | succ k => + simp [List.replicate_succ] at hacc' + subst hacc' + rw [push_dotdot_cons] + simp only [ite_true] + refine ⟨k + 2, [], ?_, ?_, by simp⟩ + · rw [List.append_nil, + show dotdot :: dotdot :: List.replicate k dotdot = List.replicate (k + 2) dotdot + from rfl, List.reverse_replicate] + · intro hr; exact absurd (hk hr) (by simp) + | cons x rest => + have hnames' : names = rest.reverse ++ [x] := by + rw [← List.reverse_reverse names, hrev]; simp + rw [hrev] at hacc' + subst hacc' + have hx : IsName x := hnames x (by simp [hnames']) + rw [List.cons_append, push_dotdot_cons] + simp only [hx.2.2, ite_false] + refine ⟨k, rest.reverse, by simp, hk, fun e he => hnames e (by simp [hnames', he])⟩ + · rw [push_name r acc e ⟨hne.1, hne.2, h2⟩] + refine ⟨k, names ++ [e], by simp [hacc], hk, ?_⟩ + intro x hx + simp only [List.mem_append, List.mem_singleton] at hx + rcases hx with hx | rfl + · exact hnames x hx + · exact ⟨hne.1, hne.2, h2⟩ + +theorem stackOk_foldl (r : Bool) (acc es : List Element) (h : StackOk r acc) : + StackOk r (es.foldl (push r) acc) := by + induction es generalizing acc with + | nil => exact h + | cons e es ih => exact ih _ (stackOk_push r acc e h) + +theorem nf_shape (r : Bool) (es : List Element) : Shape r (nf r es) := + stackOk_foldl r [] es (stackOk_nil r) + +theorem nf_normal (r : Bool) (es : List Element) : Normal r (nf r es) := + normal_of_shape r _ (nf_shape r es) + +/-- One element pushed is reached by at most one rule. -/ +theorem steps_push (r : Bool) (acc : List Element) (e : Element) (ys : List Element) + (h : StackOk r acc) : + Steps r (acc.reverse ++ e :: ys) ((push r acc e).reverse ++ ys) := by + obtain ⟨k, names, hacc, hk, hnames⟩ := h + by_cases h1 : e = [] + · subst h1; rw [push_nil]; exact Steps.single (Step.empty _ _) + by_cases h2 : e = dot + · subst h2; rw [push_dot]; exact Steps.single (Step.dot _ _) + by_cases h3 : e = dotdot + · subst h3 + cases acc with + | nil => + rw [push_dotdot_nil] + cases r + · simpa using Steps.refl (rooted := false) (dotdot :: ys) + · simpa using Steps.single (Step.root (rooted := true) ys rfl) + | cons x acc => + rw [push_dotdot_cons] + by_cases hx : x = dotdot + · subst hx; simp only [ite_true]; simpa using Steps.refl _ + · simp only [hx, ite_false] + have hxn : IsName x := by + have hm : x ∈ List.replicate k dotdot ++ names := by rw [← hacc]; simp + rcases List.mem_append.mp hm with hm | hm + · exact absurd (List.mem_replicate.mp hm).2 hx + · exact hnames x hm + simpa using Steps.single (Step.parent acc.reverse x ys hxn.1 hxn.2.1 hxn.2.2) + · rw [push_name r acc e ⟨h1, h2, h3⟩] + simpa using Steps.refl _ + +theorem steps_foldl (r : Bool) (acc es ys : List Element) (h : StackOk r acc) : + Steps r (acc.reverse ++ es ++ ys) ((es.foldl (push r) acc).reverse ++ ys) := by + induction es generalizing acc with + | nil => simpa using Steps.refl _ + | cons e es ih => + rw [List.foldl_cons] + have h1 := steps_push r acc e (es ++ ys) h + have h2 := ih (push r acc e) (stackOk_push r acc e h) + simp only [List.append_assoc, List.cons_append] at h1 h2 ⊢ + exact Steps.trans h1 h2 + +theorem steps_nf_self (r : Bool) (es : List Element) : Steps r es (nf r es) := by + simpa [nf] using steps_foldl r [] es [] (stackOk_nil r) + +theorem nf_of_shape (r : Bool) (es : List Element) (h : Shape r es) : nf r es = es := by + obtain ⟨k, names, rfl, hk, hnames⟩ := h + unfold nf + rw [List.foldl_append] + cases r with + | false => + rw [foldl_dotdots [] k (by simp), foldl_names _ _ _ hnames] + simp + | true => + rw [hk rfl] + simp [foldl_names _ _ _ hnames] + +theorem nf_of_normal (r : Bool) (es : List Element) (h : Normal r es) : nf r es = es := + nf_of_shape r es (shape_of_normal r es h) + +/-- Whatever order the rules are applied in, they end at `nf`. -/ +theorem steps_normal_eq (r : Bool) (es es' : List Element) (h : Steps r es es') + (hn : Normal r es') : es' = nf r es := by + rw [steps_nf r es es' h, nf_of_normal r es' hn] + +end Spec.Proof diff --git a/README.md b/README.md index 67ee539..395413e 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,89 @@ -# path +# path: lexical path cleaning (Go's path.Clean), proved -A Lean implementation of lexical path cleaning, following the Go path.Clean rules. This is a Proven Commons project: it rebuilds a widely used rule in Lean and proves the new code follows its written specification. +Programs clean a path before they use it: they drop `.`, let each `..` remove the name before it, and squeeze repeated slashes. A server that joins a path from a request to its own folder relies on that cleaning to keep the request inside the folder, and a mistake there lets a request read files outside it. Go's `path.Clean` gives the rules most often copied, from Rob Pike's "Lexical File Names in Plan 9". This library is those rules, and Go's `path.Join`, in Lean, proved to follow Go's documentation. -Status: not verified. No code has landed yet. Plan issue 1 holds the first slice, and it is unclaimed. The specification has not been approved by a reviewer, so do not call this verified. +It only reads the string. It never looks at a file system, so it says nothing about symbolic links. -## First slice +Machine-checked proofs show: -Clean a POSIX path string with the four rules of Go path.Clean. +- **Clean is what Go's four rules give.** Go's documentation says Clean applies four rules "iteratively until no further processing can be done". Applied in any order, the rules end at the same elements, and `clean` gives them, written back as a path. +- **The result is clean.** It has no empty element and no `.`, and `..` only at its start, and none if the path is rooted. Cleaning it again changes nothing. +- **Join is what Go's documentation says**, and **joined paths stay inside.** A clean relative path with no `..`, joined to any folder, gives the folder's elements and then its own. Any path cleaned as a rooted path, `clean ("/" ++ u)`, and joined to a folder that is not the empty string, never leaves the folder. +- **Go's own tests** of Clean and Join give Go's results. -## Main claim +It is tested against a model of the rules and against Go, Python and Node; `tests/TEST-RESULTS.md` holds the record. It is a project of [Proven Commons](https://github.com/Proven-Commons/mission), built to the verified bar published there. -Cleaning is idempotent. The result has no "." or empty element, and ".." only at the start. A cleaned relative path that does not start with ".." stays inside any folder it is joined to. +Status: in progress. The specification has not yet been approved by a reviewer apart from its author, so this is not verified. -## Test oracles +## Use it -- Go path.Clean and its tests. -- Python posixpath.normpath. -- Node path.posix.normalize. +From Lean, on `String`: -The oracles differ on a leading "//", so the specification must choose a rule for that case and record it. +- `Path.clean : String → String`; +- `Path.join : List String → String`. -## Score +`SPEC.md` says what each does. -15 out of 18, from two scorers, in the round 3 sheet in the mission repo. The sheet is mission pull request 17. +## What is proved -## Placement note +The statements are in `Spec.lean`. The rules they use are in `Spec/Defs.lean`, written from Go's documentation, and the proofs are in `Proofs/`. -The round 3 sheet raised the alternative of a module imported by the tar and zip projects, which each have their own path gate today. This repo was created at the org owner's direction, so the placement is a repo, not a module. +| Claim | Theorem | +|-------|---------| +| Whatever order Go's rules are applied in to the elements of a path, once none applies, the path they leave is `clean p`. | `Spec.clean_rules` | +| The rules can always be applied until none applies. | `Spec.rules_end` | +| No rule applies exactly when the elements are some `..`, none if the path is rooted, then names. | `Spec.normal_iff` | +| `clean p` is such a path, and rooted exactly when `p` is. | `Spec.clean_names` | +| Cleaning a clean path changes nothing. | `Spec.clean_idem` | +| `join` leaves out empty elements, joins the rest with slashes and cleans them, and gives the empty string when nothing is left. | `Spec.join_eq` | +| A clean relative path with no `..`, joined to any path `b`, gives the elements of `clean b`, then its own. | `Spec.join_inside` | +| Any path cleaned as rooted, joined to a path `b` that is not empty, gives the elements of `clean b`, then elements with no `..`. | `Spec.join_root_inside` | +| Go's tests of Clean, `cleantests`: each path cleans to its result, and the result to itself. Go's tests of Join, `jointests`. | `Spec.go_clean_tests`, `Spec.go_join_tests` | -## What is not in this slice +## What is not proved -- Filesystem access. This is string cleaning only. -- Windows path rules. -- Any claim about a full filesystem implementation. +- **The file system.** The rules are lexical. If `b/x` is a symbolic link, `b/x/..` on the file system need not be `b`, and a link inside a folder can point outside it. +- **Two slashes at the start.** POSIX lets a system give `//` at the start of a path a meaning of its own. Go, and this library, read it as `/`; Python keeps it. That is a choice, recorded in `SPEC.md`. +- **The empty folder.** `join ["", u]` leaves the empty element out, so `join ["", clean ("/" ++ u)]` is rooted. `join_root_inside` needs a folder that is not empty. +- **Other path functions** of Go: `Split`, `Base`, `Dir`, `Ext`, `IsAbs` and `Match`. Windows paths are not covered. +- **Strings that are not UTF-8.** Lean strings are Unicode text, so a Go string holding other octets cannot be passed. +- **Time and memory.** No proof bounds them. `clean` reads the path once, as Go's loop does, and removes each character it writes at most once; the tests clean paths of a million characters. +- **Reading requests and printing answers in `Main.lean`.** The differential tests cover them. -## Layout, when the first slice lands +## What you must trust -The layout follows the Proven Commons template in the mission repo, under bar/template. Spec.lean holds the statements a reviewer approves. Proofs.lean holds the proofs under the same names. Main.lean holds the command line tool used by the tests. The bar check runs in CI on every change. +- The Lean kernel and the five external kernels shipped with Lean, which all check the proofs. +- `lake comparator`, which builds the statements and the proofs in a sandbox, exports them, and checks that each proof proves exactly its statement. +- bubblewrap and the Linux kernel's namespaces, which keep the untrusted build from tampering with that check. +- The Lean compiler, with the C compiler and linker shipped with it, and the runtime, which turn the proved definitions into the program that runs. +- Lean's core library, including the compiled code it puts in place of some definitions the proofs reason about, such as natural number arithmetic and the conversion between strings and their characters. +- The operating system and hardware it runs on. +- For the tests only: the model in `tests/run_tests.py`, and the implementations it compares against. The record names their versions. + +This project has no dependencies beyond Lean itself. + +## Re-check it + +Needs Linux, [elan](https://github.com/leanprover/elan), bubblewrap (`apt install bubblewrap`) and Python 3. The oracles also need Go and Node; the test script leaves out any it cannot start, and the record names them. + +``` +lake build +lake comparator --paranoid +python3 tests/run_tests.py +python3 tests/plant_bugs.py +``` + +`lake comparator --paranoid` builds the statements and the proofs in a sandbox. It checks that each theorem in `Proofs/` proves exactly the statement in `Spec.lean`, using only the axioms `propext`, `Classical.choice`, and `Quot.sound`, and replays it through every kernel shipped with Lean. + +## Layout + +- `Path/Basic.lean`: the code under proof, Go's loop written in Lean. +- `Spec.lean`: the statements, with `sorry` for proofs. A reviewer approves this file. +- `Spec/Defs.lean`: the rules the statements use, written from Go's documentation. +- `Proofs.lean` and `Proofs/`: the proofs. +- `Main.lean`: the command-line tool the tests drive. +- `lakefile.toml`: the build. It names the package and its version, its libraries and executable, and the default targets, and sets nothing else. +- `comparator.json`: which theorems `lake comparator` checks, and which axioms it allows. +- `bar.toml`: the record for the bar check: status, authors, approvals, and where things are. +- `SPEC.md`: the specification in plain words. +- `tests/`: the differential tests, the oracles, and the planted bugs. diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..5f7e1c9 --- /dev/null +++ b/SPEC.md @@ -0,0 +1,76 @@ +# Specification: path + +Go's `path.Clean` (https://pkg.go.dev/path#Clean) gives "the shortest path name equivalent to path by purely lexical processing". It reads only the string: it never looks at a file system. This library does the same, and Go's `path.Join`, in Lean, on strings. + +A reviewer who did not write this specification approves it. The approval also covers `Spec.lean`, `Spec/Defs.lean`, and the other files that `bar.toml` lists. + +## Elements + +`Spec/Defs.lean` states Go's rules on the elements of a path: the pieces between its slashes. `Spec.Splits p es` says that `p` is the elements `es` with a slash between each two, and that no element holds a slash. A path has at least one element; the empty path has the one empty element. + +A path is rooted when it starts with a slash (`Spec.Rooted`). The root is kept apart from the elements, so `/a/b` is rooted, with the elements "", `a` and `b`. An empty element comes from two slashes in a row, from the slash of a rooted path, or from a slash at the end. + +## The rules + +Go's documentation says that Clean "applies the following rules iteratively until no further processing can be done": + +1. Replace multiple slashes with a single slash. +2. Eliminate each `.` path name element (the current directory). +3. Eliminate each inner `..` path name element (the parent directory) along with the non-`..` element that precedes it. +4. Eliminate `..` elements that begin a rooted path: that is, replace `/..` by `/` at the beginning of a path. + +It adds: "The returned path ends in a slash only if it is the root `/`", and "If the result of this process is an empty string, Clean returns the string `.`". + +`Spec.Step` is one rule, applied once, to the elements: + +- **Rule 1** drops an empty element. On elements, rule 1 and the sentence on a slash at the end say the same thing. +- **Rule 2** drops a `.` element. +- **Rule 3** drops a `..` element and the element before it, when that is a name: not empty, not `.` and not `..`. An empty element or a `.` before the `..` is dropped first, by rule 1 or 2. +- **Rule 4** drops a `..` at the start of a rooted path. + +`Spec.Steps` is the rules applied any number of times, in any order. `Spec.Normal` says that no rule applies. `Spec.render` writes elements back as a path: a slash first when it is rooted, a slash between each two elements, and `.` for no elements in a path that is not rooted. So the result ends in a slash only when it is the root `/`. + +**Clean** is the path `render` writes for the elements the rules leave, in whatever order they are applied (`clean_rules`), and the rules can always be applied until none applies (`rules_end`). So the order does not matter: every order ends at the same elements. + +No rule applies exactly when the elements are some `..`, none if the path is rooted, and then names (`normal_iff`). So the result of `clean` has no empty element and no `.`, and `..` only at its start, and none if it is rooted. It is rooted exactly when the path was (`clean_names`). Cleaning it again changes nothing (`clean_idem`). + +**Two slashes at the start.** POSIX says that "a pathname that begins with two successive slashes may be interpreted in an implementation-defined manner". Go reads them as one slash, and so does this library: `//abc` cleans to `/abc`, as in Go's own tests. Python's `posixpath.normpath` keeps exactly two. + +## Join + +Go's documentation: "Join joins any number of path elements into a single path, separating them with slashes. Empty elements are ignored. The result is Cleaned. However, if the argument list is empty or all its elements are empty, Join returns an empty string." `join_eq` states this. + +**Joined paths stay inside.** A clean relative path `q` with no `..`, joined to any path `b`, gives the elements of `clean b` and then those of `q` (`join_inside`). Any path `u`, cleaned as a rooted path (`clean ("/" ++ u)`) and joined to a path `b` that is not empty, gives the elements of `clean b` and then elements with no `..` (`join_root_inside`). If `b` is empty, `join` leaves it out, and the result is rooted: `join ["", "/etc"]` is `/etc`. + +"Inside" is lexical: it says what the strings are, not where a file system resolves them. A symbolic link inside `b` can point anywhere. + +## What must always hold + +Each property is a theorem in `Spec.lean`, under the name given. + +| Property | Theorem | +|----------|---------| +| Whatever order the rules are applied in, once none applies, `render` gives `clean p`. | `Spec.clean_rules` | +| The rules can always be applied until none applies. | `Spec.rules_end` | +| No rule applies exactly when the elements are some `..` (none if rooted), then names. | `Spec.normal_iff` | +| `clean p` is `render` of such elements, rooted when `p` is. | `Spec.clean_names` | +| Cleaning a clean path changes nothing. | `Spec.clean_idem` | +| `join` is what Go's documentation says. | `Spec.join_eq` | +| A clean relative path with no `..`, joined to `b`, adds its elements to those of `clean b`. | `Spec.join_inside` | +| Any path cleaned as rooted, joined to a `b` that is not empty, adds elements with no `..` to those of `clean b`. | `Spec.join_root_inside` | +| Go's tests of Clean, each path and each result. | `Spec.go_clean_tests` | +| Go's tests of Join, and an empty first element joined to a rooted path. | `Spec.go_join_tests` | + +## Strings + +The functions take and give Lean strings, which are Unicode text in UTF-8. Go's strings may hold any octets. The rules look only at `/` and `.`, which are single octets in UTF-8 and never part of another character, so Go should give the same answer on UTF-8 text; the tests check this with characters that are not ASCII. A Go string that is not UTF-8 cannot be passed to this library. + +## The command-line tool + +`Main.lean` has these modes: + +- `path clean PATH` prints the path cleaned. +- `path join ELEM...` prints the elements joined and cleaned, and an empty line for no elements or only empty ones. +- `path serve` answers requests from the differential tests, one per line. The header comment of `Main.lean` gives the protocol. + +Wrong arguments exit with 2. Every answer comes from the functions above. diff --git a/Spec.lean b/Spec.lean new file mode 100644 index 0000000..938597c --- /dev/null +++ b/Spec.lean @@ -0,0 +1,117 @@ +import Path +import Spec.Defs + +/-! + The statements a reviewer approves. Each `sorry` is proved under the + same name in `Proofs/`, and `lake comparator` checks that each proof + proves exactly the statement here. + + `Spec/Defs.lean` states Go's rules on the elements of a path. + `Path.clean` and `Path.join` are the code, in `Path/Basic.lean`. +-/ + +namespace Spec + +/-- Whatever order the rules are applied in, once none applies, the + path with the elements left is `clean p`. -/ +theorem clean_rules (p : String) (es es' : List Element) + (hp : Splits p.toList es) (hs : Steps (Rooted p.toList) es es') + (hn : Normal (Rooted p.toList) es') : + (Path.clean p).toList = render (Rooted p.toList) es' := by + sorry + +/-- The rules can always be applied until none applies, so + `clean_rules` says something about every path. -/ +theorem rules_end (p : String) : + ∃ es es', Splits p.toList es ∧ Steps (Rooted p.toList) es es' ∧ + Normal (Rooted p.toList) es' := by + sorry + +/-- No rule applies exactly when the elements are some `..`, none if + the path is rooted, then names: elements that are not empty, `.` + or `..`. -/ +theorem normal_iff (rooted : Bool) (es : List Element) : + Normal rooted es ↔ + ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ + ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := by + sorry + +/-- So `clean p` has no empty element and no `.`, `..` only at the + start and none if `p` is rooted, and it is rooted when `p` is. -/ +theorem clean_names (p : String) : ∃ es, Names (Path.clean p) (Rooted p.toList) es := by + sorry + +/-- Cleaning a clean path changes nothing. -/ +theorem clean_idem (p : String) : Path.clean (Path.clean p) = Path.clean p := by + sorry + +/-- `join`, as Go's documentation says: "Join joins any number of path + elements into a single path, separating them with slashes. Empty + elements are ignored. The result is Cleaned. However, if the + argument list is empty or all its elements are empty, Join returns + an empty string." -/ +theorem join_eq (es : List String) : + Path.join es = + if es.all (· = "") then "" else Path.clean ("/".intercalate (es.filter (· ≠ ""))) := by + sorry + +/-- A clean relative path `q` with no `..`, joined to any path `b`, + stays inside `b`: the elements of the result are those of + `clean b`, then those of `q`. -/ +theorem join_inside (b q : String) (eq : List Element) (hq : Names q false eq) + (hd : dotdot ∉ eq) : + ∃ eb, Names (Path.clean b) (Rooted b.toList) eb ∧ + Names (Path.join [b, q]) (Rooted b.toList) (eb ++ eq) := by + sorry + +/-- Any path `u`, cleaned as a rooted path and joined to a path `b` + that is not empty, stays inside `b`. If `b` is empty, `join` leaves + it out, and the result is rooted (see `go_join_tests`). -/ +theorem join_root_inside (b u : String) (hb : b ≠ "") : + ∃ eb ext, dotdot ∉ ext ∧ Names (Path.clean b) (Rooted b.toList) eb ∧ + Names (Path.join [b, Path.clean ("/" ++ u)]) (Rooted b.toList) (eb ++ ext) := by + sorry + +/-- Go's tests of `Clean`, `cleantests` in src/path/path_test.go + (go1.24.7): each path cleans to its result, and the result cleans + to itself. Among them, `//abc` cleans to `/abc`: two slashes at the + start are one slash, though POSIX lets a system give them a meaning + of their own. -/ +theorem go_clean_tests : + ∀ t ∈ [ + -- Already clean + ("", "."), ("abc", "abc"), ("abc/def", "abc/def"), ("a/b/c", "a/b/c"), + (".", "."), ("..", ".."), ("../..", "../.."), ("../../abc", "../../abc"), + ("/abc", "/abc"), ("/", "/"), + -- Remove trailing slash + ("abc/", "abc"), ("abc/def/", "abc/def"), ("a/b/c/", "a/b/c"), ("./", "."), + ("../", ".."), ("../../", "../.."), ("/abc/", "/abc"), + -- Remove doubled slash + ("abc//def//ghi", "abc/def/ghi"), ("//abc", "/abc"), ("///abc", "/abc"), + ("//abc//", "/abc"), ("abc//", "abc"), + -- Remove . elements + ("abc/./def", "abc/def"), ("/./abc/def", "/abc/def"), ("abc/.", "abc"), + -- Remove .. elements + ("abc/def/ghi/../jkl", "abc/def/jkl"), ("abc/def/../ghi/../jkl", "abc/jkl"), + ("abc/def/..", "abc"), ("abc/def/../..", "."), ("/abc/def/../..", "/"), + ("abc/def/../../..", ".."), ("/abc/def/../../..", "/"), + ("abc/def/../../../ghi/jkl/../../../mno", "../../mno"), + -- Combinations + ("abc/./../def", "def"), ("abc//./../def", "def"), + ("abc/../../././../def", "../../def")], + Path.clean t.1 = t.2 ∧ Path.clean t.2 = t.2 := by + sorry + +/-- Go's tests of `Join`, `jointests` in src/path/path_test.go + (go1.24.7), and one more: an empty first element is left out, so + joining it to a rooted path gives a rooted path. -/ +theorem go_join_tests : + ∀ t ∈ [ + ([], ""), ([""], ""), (["a"], "a"), + (["a", "b"], "a/b"), (["a", ""], "a"), (["", "b"], "b"), (["/", "a"], "/a"), + (["/", ""], "/"), (["a/", "b"], "a/b"), (["a/", ""], "a"), (["", ""], ""), + (["", "/etc"], "/etc")], + Path.join t.1 = t.2 := by + sorry + +end Spec diff --git a/Spec/Defs.lean b/Spec/Defs.lean new file mode 100644 index 0000000..c56e06c --- /dev/null +++ b/Spec/Defs.lean @@ -0,0 +1,96 @@ +/- + The rules of Go's `path.Clean`, written from its documentation + (https://pkg.go.dev/path#Clean, go1.24.7): + + Clean returns the shortest path name equivalent to path by purely + lexical processing. It applies the following rules iteratively + until no further processing can be done: + + 1. Replace multiple slashes with a single slash. + 2. Eliminate each . path name element (the current directory). + 3. Eliminate each inner .. path name element (the parent + directory) along with the non-.. element that precedes it. + 4. Eliminate .. elements that begin a rooted path: that is, + replace "/.." by "/" at the beginning of a path. + + The returned path ends in a slash only if it is the root "/". + + If the result of this process is an empty string, Clean returns + the string ".". + + and from Rob Pike, "Lexical File Names in Plan 9, or Getting Dot-Dot + Right" (https://9p.io/sys/doc/lexnames.html), which it cites. + + The rules are stated here on the elements of a path, the pieces + between its slashes. A path is rooted when it starts with a slash. + The root is kept apart, so the elements of `/a/b` are the empty + element before the first slash, `a` and `b`, and the path is rooted. + An empty element comes from two slashes in a row, from the slash of + a rooted path, or from a slash at the end. Rule 1 and the sentence + on a slash at the end both say the same of elements: an empty element + is dropped. + + This file uses nothing from the code. +-/ + +namespace Spec + +/-- A path element: the characters between two slashes. -/ +abbrev Element := List Char + +/-- The element `.`. -/ +def dot : Element := ['.'] + +/-- The element `..`. -/ +def dotdot : Element := ['.', '.'] + +/-- `es` are the elements of the path `p`: `p` is the elements with a + slash between each two, and no element holds a slash. There is at + least one element; the path "" has the one element "". -/ +def Splits (p : List Char) (es : List Element) : Prop := + es ≠ [] ∧ p = List.intercalate ['/'] es ∧ ∀ e ∈ es, '/' ∉ e + +/-- A path is rooted when it starts with a slash. -/ +def Rooted (p : List Char) : Bool := + p.head? == some '/' + +/-- One rule, applied once, to the elements of a path that is `rooted` + or not. -/ +inductive Step (rooted : Bool) : List Element → List Element → Prop + /-- Rule 1, with the sentence on a slash at the end: drop an empty + element. -/ + | empty (xs ys : List Element) : Step rooted (xs ++ [] :: ys) (xs ++ ys) + /-- Rule 2: drop a `.` element. -/ + | dot (xs ys : List Element) : Step rooted (xs ++ dot :: ys) (xs ++ ys) + /-- Rule 3: drop a `..` element and the element before it, when that + is a name: not empty, not `.` and not `..`. -/ + | parent (xs : List Element) (e : Element) (ys : List Element) : + e ≠ [] → e ≠ dot → e ≠ dotdot → + Step rooted (xs ++ e :: dotdot :: ys) (xs ++ ys) + /-- Rule 4: drop a `..` element at the start of a rooted path. -/ + | root (ys : List Element) : rooted = true → Step rooted (dotdot :: ys) ys + +/-- The rules, applied any number of times, in any order. -/ +inductive Steps (rooted : Bool) : List Element → List Element → Prop + | refl (es : List Element) : Steps rooted es es + | step {es es' es'' : List Element} : + Step rooted es es' → Steps rooted es' es'' → Steps rooted es es'' + +/-- No rule applies: "no further processing can be done". -/ +def Normal (rooted : Bool) (es : List Element) : Prop := + ∀ es', ¬ Step rooted es es' + +/-- The path with these elements: a slash first when it is rooted, and + a slash between each two elements. So it ends in a slash only when + it is the root `/`. An empty path that is not rooted is `.`. -/ +def render (rooted : Bool) (es : List Element) : List Char := + if rooted then '/' :: List.intercalate ['/'] es + else if es = [] then dot + else List.intercalate ['/'] es + +/-- `s` is the path, rooted or not, with the elements `es`: no element + holds a slash, and no rule applies to them. -/ +def Names (s : String) (rooted : Bool) (es : List Element) : Prop := + s.toList = render rooted es ∧ Normal rooted es ∧ ∀ e ∈ es, '/' ∉ e + +end Spec diff --git a/bar.toml b/bar.toml new file mode 100644 index 0000000..6b1c6a6 --- /dev/null +++ b/bar.toml @@ -0,0 +1,33 @@ +# Read by bar/check.py in Proven-Commons/mission. +# See playbook/verified-bar.md there for what each field means. + +[project] +name = "path" +# "in-progress" until every bar item passes, then "verified". +status = "in-progress" +page = "README.md" +# GitHub handles of everyone who wrote the specification. It was drafted +# by Claude, an AI agent, for the org: agent fdn-claude, for the fdn-0 +# account, posting as zero-aska on GitHub. +spec_authors = ["claude", "fdn-0", "zero-aska"] +# The program people run. A change to it voids the approval. +entry_points = ["Main.lean"] + +[spec] +files = ["SPEC.md", "Spec.lean", "Spec/Defs.lean"] +# The code under proof: the functions the statements are about. +code = ["Path.clean", "Path.join"] + +# One entry per approval, added in a pull request the reviewer approves. +# [[spec.approval]] +# reviewer = "github-handle" +# date = "2026-11-02" +# sha256 = "..." +# record = "https://github.com/Proven-Commons/path/pull/" + +[tests] +results = "tests/TEST-RESULTS.md" +command = "python3 tests/run_tests.py" +files = ["tests/run_tests.py", "tests/plant_bugs.py", "tests/oracles/go/main.go", + "tests/oracles/go/go.mod", "tests/oracles/python/oracle.py", + "tests/oracles/node/oracle.js"] diff --git a/comparator.json b/comparator.json new file mode 100644 index 0000000..c147aad --- /dev/null +++ b/comparator.json @@ -0,0 +1,10 @@ +{ + "challenge_module": "Spec", + "solution_module": "Proofs", + "theorem_names": [ + "Spec.clean_rules", "Spec.rules_end", "Spec.normal_iff", "Spec.clean_names", + "Spec.clean_idem", "Spec.join_eq", "Spec.join_inside", "Spec.join_root_inside", + "Spec.go_clean_tests", "Spec.go_join_tests" + ], + "permitted_axioms": ["propext", "Quot.sound", "Classical.choice"] +} diff --git a/lake-manifest.json b/lake-manifest.json new file mode 100644 index 0000000..2e16cf0 --- /dev/null +++ b/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.3.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "path", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/lakefile.toml b/lakefile.toml new file mode 100644 index 0000000..f593299 --- /dev/null +++ b/lakefile.toml @@ -0,0 +1,16 @@ +name = "path" +version = "0.1.0" +defaultTargets = ["Path", "Spec", "Proofs", "path"] + +[[lean_lib]] +name = "Path" + +[[lean_lib]] +name = "Spec" + +[[lean_lib]] +name = "Proofs" + +[[lean_exe]] +name = "path" +root = "Main" diff --git a/lean-toolchain b/lean-toolchain new file mode 100644 index 0000000..a6aff06 --- /dev/null +++ b/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.35.0 diff --git a/tests/PLANTED-BUGS.md b/tests/PLANTED-BUGS.md new file mode 100644 index 0000000..09794da --- /dev/null +++ b/tests/PLANTED-BUGS.md @@ -0,0 +1,31 @@ +# Planted bugs + +Written by `tests/plant_bugs.py`. Each bug is a change of one line, or of two, to `Path/Basic.lean` unless it names `Main.lean`, tested with `tests/run_tests.py --count 1000 --no-oracles`, which builds only the executable and compares it with the model alone. Every bug must give failures, and the unchanged code none. A bug that does not build, or a run that gives up or dies, counts as missed. + +| Planted bug | Failures | +|---|---| +| none | 0 | +| backtracking removes one character more, past `dotdot` | 1281 | +| backtracking does not stop at a slash | 2121 | +| `..` backtracks when the output is no longer than `dotdot` | 4842 | +| a `..` that cannot backtrack is kept in a rooted path, not in one that is not | 6537 | +| `dotdot` does not move past a `..` that is kept | 289 | +| a kept `..` gets a slash before it even at the start | 4426 | +| a `.` at the end is kept | 10537 | +| a `..` at the end is dropped | 3410 | +| a `.` before a slash is copied as a name | 29229 | +| in a rooted path, the first name gets a slash after the root | 32251 | +| a name runs on past the slash after it | 68609 | +| the length is not counted while a name is copied | 3347 | +| the root's `dotdot` is 0, so a `..` can remove the root | 1653 | +| the root alone is not rooted | 15 | +| an empty result is the empty string, not `.` | 2320 | +| `join` of only empty elements gives `.` | 129 | +| `join` keeps the empty elements before the first other one | 82 | +| Main.lean: `-` is read as the path `-` | 357 | +| Main.lean: `clean` takes more than one field | 2 | +| Main.lean: a CR before the LF is kept | 15536 | +| Main.lean: `path clean` prints the path as given | 4 | +| Main.lean: wrong arguments exit with 0 | 6 | + +Bugs caught: 22 of 22 diff --git a/tests/TEST-RESULTS.md b/tests/TEST-RESULTS.md new file mode 100644 index 0000000..65fb798 --- /dev/null +++ b/tests/TEST-RESULTS.md @@ -0,0 +1,39 @@ +# Test results + +Written by `tests/run_tests.py`. Rerun it to check these numbers. + +- Lean: Lean (version 4.35.0, x86_64-unknown-linux-gnu, commit eb6cafb7290e8bd5f2937522d8d804f58f02bafc, Release) +- Oracle: the model of Go's rules in `tests/run_tests.py`; Go: `path.Clean` and `path.Join` of the standard library of go version go1.24.7 linux/amd64; Python: `posixpath.normpath` of Python 3.13.17; Node: `path.posix.normalize` and `path.posix.join` of Node v22.22.0 +- Go: `path.Clean` and `path.Join` of the standard library of go version go1.24.7 linux/amd64 +- Python: `posixpath.normpath` of Python 3.13.17 +- Node: `path.posix.normalize` and `path.posix.join` of Node v22.22.0 +- Oracles left out: none +- Seed: 20261027 +- Inputs: 167263 +- By kind: Go cleantests (clean) 36, Go cleantests, the result (clean) 36, Go jointests (join) 12, Pike, and edges (clean) 29, every path over a . / up to 10 (clean) 88573, every path over a b . / up to 7 (clean) 18565, random path (clean) 40000, random join (join) 20000, long (clean) 11, long (join) 1 +- Requests: `clean` 147250, `join` 20013 +- The model on Go's `cleantests` and `jointests`: 47 of 47 +- The model's two parts, the rules applied in a random order and the stack, agree on 147239 of 147239 paths +- Answers that differ from the model: 0 +- Long inputs: 12, 12296441 characters in all, answered in 3.3 s by one `path serve` +- Agreement with each oracle over the requests it answers: Go 167263 of 167263, Python 138020 of 147250, Node 117446 of 167263 +- Requests each oracle answers, by operation: Go: clean 147250, join 20013; Python: clean 147250; Node: clean 147250, join 20013 +- Malformed requests answered `error`: 31 of 31 +- Requests ending in CR LF: every seventh, 23893 +- Last lines sent alone, with no LF, with CR LF, or with a lone CR, answered right: 4 of 4 +- Command-line modes (`clean`, `join`, and wrong arguments), printing and exiting as the model says: 18 of 18 +- Failures: 0 + +## Where the oracles differ + +Each disagreement is counted under the reason the model confirms, with the first input that showed it. + +### Python + +- Python keeps two slashes at the start of a path, as POSIX allows: 9230; for example `clean 2f2f616263` + +### Node + +- Node keeps a slash at the end of a path: 41482; for example `clean 6162632f` +- Node's `join` gives `.` when every element is empty, or there are none: 5099; for example `join` +- Node keeps a slash at the end of a path (in `join`): 3236; for example `join 612f -` diff --git a/tests/oracles/go/go.mod b/tests/oracles/go/go.mod new file mode 100644 index 0000000..b8e4a26 --- /dev/null +++ b/tests/oracles/go/go.mod @@ -0,0 +1,3 @@ +module pathoracle + +go 1.24 diff --git a/tests/oracles/go/main.go b/tests/oracles/go/main.go new file mode 100644 index 0000000..dc9dbbc --- /dev/null +++ b/tests/oracles/go/main.go @@ -0,0 +1,75 @@ +// The Go oracle: path.Clean and path.Join from the standard library. +// +// It reads requests on standard input, one per line, as `path serve` +// does (see the header of Main.lean): +// +// clean HEX prints OK and the hex of path.Clean +// join HEX... prints OK and the hex of path.Join +// +// A path is the hex of its octets, and the empty path is `-`. Anything +// else prints `n/a`. +package main + +import ( + "bufio" + "encoding/hex" + "fmt" + "os" + "path" + "strings" +) + +func field(f string) (string, bool) { + if f == "-" { + return "", true + } + b, err := hex.DecodeString(f) + if err != nil || f == "" { + return "", false + } + return string(b), true +} + +func out(s string) string { + if s == "" { + return "-" + } + return hex.EncodeToString([]byte(s)) +} + +func answer(line string) string { + fs := strings.Split(line, " ") + switch { + case fs[0] == "clean" && len(fs) == 2: + p, ok := field(fs[1]) + if !ok { + return "n/a" + } + return "OK " + out(path.Clean(p)) + case fs[0] == "join": + var es []string + for _, f := range fs[1:] { + e, ok := field(f) + if !ok { + return "n/a" + } + es = append(es, e) + } + return "OK " + out(path.Join(es...)) + } + return "n/a" +} + +func main() { + in := bufio.NewReaderSize(os.Stdin, 1<<20) + w := bufio.NewWriterSize(os.Stdout, 1<<20) + defer w.Flush() + for { + line, err := in.ReadString('\n') + if line == "" && err != nil { + return + } + line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + fmt.Fprintln(w, answer(line)) + } +} diff --git a/tests/oracles/node/oracle.js b/tests/oracles/node/oracle.js new file mode 100644 index 0000000..ad0dc9f --- /dev/null +++ b/tests/oracles/node/oracle.js @@ -0,0 +1,66 @@ +// The Node oracle: path.posix.normalize and path.posix.join. +// +// It reads requests on standard input, one per line, as `path serve` +// does (see the header of Main.lean): +// +// clean HEX prints OK and the hex of path.posix.normalize +// join HEX... prints OK and the hex of path.posix.join +// +// A path is the hex of its UTF-8 octets, and the empty path is `-`. +// Anything else prints `n/a`. +'use strict'; +const path = require('path'); + +function field(f) { + if (f === '-') return ''; + if (f === '' || f.length % 2 !== 0 || !/^[0-9a-f]*$/.test(f)) return null; + const b = Buffer.from(f, 'hex'); + const s = b.toString('utf8'); + // Only valid UTF-8 is asked; refuse anything that does not round-trip. + if (!Buffer.from(s, 'utf8').equals(b)) return null; + return s; +} + +function out(s) { + return s === '' ? '-' : Buffer.from(s, 'utf8').toString('hex'); +} + +function answer(line) { + const fs = line.split(' '); + if (fs[0] === 'clean' && fs.length === 2) { + const p = field(fs[1]); + return p === null ? 'n/a' : 'OK ' + out(path.posix.normalize(p)); + } + if (fs[0] === 'join') { + const es = fs.slice(1).map(field); + if (es.some((e) => e === null)) return 'n/a'; + return 'OK ' + out(path.posix.join(...es)); + } + return 'n/a'; +} + +let buf = ''; +const outBuf = []; +process.stdin.setEncoding('latin1'); +process.stdin.on('data', (chunk) => { + buf += chunk; + let i; + while ((i = buf.indexOf('\n')) >= 0) { + let line = buf.slice(0, i); + buf = buf.slice(i + 1); + if (line.endsWith('\r')) line = line.slice(0, -1); + outBuf.push(answer(line)); + if (outBuf.length >= 4096) { + process.stdout.write(outBuf.join('\n') + '\n'); + outBuf.length = 0; + } + } +}); +process.stdin.on('end', () => { + if (buf.length > 0) { + let line = buf; + if (line.endsWith('\r')) line = line.slice(0, -1); + outBuf.push(answer(line)); + } + if (outBuf.length > 0) process.stdout.write(outBuf.join('\n') + '\n'); +}); diff --git a/tests/oracles/python/oracle.py b/tests/oracles/python/oracle.py new file mode 100644 index 0000000..fb8ff1b --- /dev/null +++ b/tests/oracles/python/oracle.py @@ -0,0 +1,49 @@ +"""The Python oracle: posixpath.normpath from the standard library. + +It reads requests on standard input, one per line, as `path serve` +does (see the header of Main.lean), and answers `clean HEX` with OK and +the hex of posixpath.normpath. posixpath.join keeps only the last +rooted argument, which is not what Go's Join does, so `join` and +anything else get `n/a`. A path is the hex of its UTF-8 octets, and +the empty path is `-`. +""" + +import posixpath +import sys + + +def field(f): + if f == "-": + return "" + if not f: + return None + try: + return bytes.fromhex(f).decode("utf-8") + except ValueError: + return None + + +def out(s): + return s.encode("utf-8").hex() or "-" + + +def answer(line): + fs = line.split(" ") + if fs[0] == "clean" and len(fs) == 2: + p = field(fs[1]) + if p is not None: + return "OK " + out(posixpath.normpath(p)) + return "n/a" + + +def main(): + stdin = sys.stdin.buffer + stdout = sys.stdout + for raw in stdin: + line = raw.decode("utf-8", "replace").rstrip("\n").rstrip("\r") + stdout.write(answer(line) + "\n") + stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/tests/plant_bugs.py b/tests/plant_bugs.py new file mode 100644 index 0000000..db2e89e --- /dev/null +++ b/tests/plant_bugs.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Check that the differential tests catch planted bugs. + +Each bug is a change of one line, or of two, to Path/Basic.lean, or to +Main.lean where its description says so. The script copies the +project to a temporary folder, plants one bug at a time, runs +tests/run_tests.py there with a small count and without the oracles, +and records how many failures the tests report. So every failure +counted is the tool disagreeing with the model of Go's rules; an +oracle that crashes or times out cannot pass for a catch. Every +planted bug must give at least one failure. The unchanged copy must +give none. A bug that stops the executable from building is not a +catch: the tests then give up, and the bug counts as missed. So is any +run that gives up (its record shows `- Failures: 1 (why)`), a run that +dies without writing its record, and one whose exit status does not +match the failures it records. The tests build only the executable +(`lake build path`), so a bug that breaks a proof still runs. + + python3 tests/plant_bugs.py [--count N] + +The script writes tests/PLANTED-BUGS.md and exits non-zero if any bug +goes uncaught. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RECORD = ROOT / "tests" / "PLANTED-BUGS.md" +CODE = "Path/Basic.lean" + +# (what the bug does, the text as written, the text with the bug[, file]) +BUGS = [ + ("backtracking removes one character more, past `dotdot`", + " | c :: out, w + 1 => if w > dotdot ∧ c ≠ '/' then backtrack dotdot out w else (out, w)", + " | c :: out, w + 1 => if w ≥ dotdot ∧ c ≠ '/' then backtrack dotdot out w else (out, w)"), + ("backtracking does not stop at a slash", + " | c :: out, w + 1 => if w > dotdot ∧ c ≠ '/' then backtrack dotdot out w else (out, w)", + " | c :: out, w + 1 => if w > dotdot then backtrack dotdot out w else (out, w)"), + ("`..` backtracks when the output is no longer than `dotdot`", + " if w > dotdot then\n", + " if w ≥ dotdot then\n"), + ("a `..` that cannot backtrack is kept in a rooted path, not in one that is not", + " else if !rooted then", + " else if rooted then"), + ("`dotdot` does not move past a `..` that is kept", + " ('.' :: '.' :: out, w + 2, w + 2)", + " ('.' :: '.' :: out, w + 2, dotdot)"), + ("a kept `..` gets a slash before it even at the start", + " let (out, w) := if w > 0 then ('/' :: out, w + 1) else (out, w)", + " let (out, w) := if w ≥ 0 then ('/' :: out, w + 1) else (out, w)"), + ("a `.` at the end is kept", + " | ['.'] => (out, w)", + " | ['.'] => ('.' :: out, w + 1)"), + ("a `..` at the end is dropped", + " let (out, w, _) := parent rooted dotdot out w", + " let (out, w, _) := (out, w, dotdot)"), + ("a `.` before a slash is copied as a name", + " | '.' :: rest@('/' :: _) => loop rooted dotdot out w rest", + " | '.' :: rest@('/' :: _) => copy rooted dotdot ('.' :: out) (w + 1) rest"), + ("in a rooted path, the first name gets a slash after the root", + " if rooted && w != 1 || !rooted && w != 0 then ('/' :: out, w + 1) else (out, w)", + " if rooted && w != 0 || !rooted && w != 0 then ('/' :: out, w + 1) else (out, w)"), + ("a name runs on past the slash after it", + " | [] => (out, w)\n | '/' :: rest => loop rooted dotdot out w rest", + " | [] => (out, w)\n | '/' :: rest => copy rooted dotdot ('/' :: out) (w + 1) rest"), + ("the length is not counted while a name is copied", + " | c :: rest => copy rooted dotdot (c :: out) (w + 1) rest", + " | c :: rest => copy rooted dotdot (c :: out) w rest"), + ("the root's `dotdot` is 0, so a `..` can remove the root", + " let (out, w) := if rooted then loop true 1 ['/'] 1 rest else loop false 0 [] 0 path", + " let (out, w) := if rooted then loop true 0 ['/'] 1 rest else loop false 0 [] 0 path"), + ("the root alone is not rooted", + " let rooted := c == '/'", + " let rooted := c == '/' && rest != []"), + ("an empty result is the empty string, not `.`", + " if w == 0 then ['.'] else out.reverse", + " if w == 0 then [] else out.reverse"), + ("`join` of only empty elements gives `.`", + " if elem.all (·.isEmpty) then \"\"", + " if elem.all (·.isEmpty) then \".\""), + ("`join` keeps the empty elements before the first other one", + " else clean (\"/\".intercalate (elem.dropWhile (·.isEmpty)))", + " else clean (\"/\".intercalate elem)"), + ("Main.lean: `-` is read as the path `-`", + " if f = \"-\" then some \"\"", + " if f = \"-\" then some \"-\"", + "Main.lean"), + ("Main.lean: `clean` takes more than one field", + " | [\"clean\", h] => do", + " | \"clean\" :: h :: _ => do", + "Main.lean"), + ("Main.lean: a CR before the LF is kept", + " String.ofList (if cs.getLast? = some '\\r' then cs.dropLast else cs)", + " String.ofList cs", + "Main.lean"), + ("Main.lean: `path clean` prints the path as given", + " | [\"clean\", p] => IO.println (Path.clean p); return 0", + " | [\"clean\", p] => IO.println p; return 0", + "Main.lean"), + ("Main.lean: wrong arguments exit with 0", + " | _ => IO.eprintln usage; return 2", + " | _ => IO.eprintln usage; return 0", + "Main.lean"), +] + + +def failures(copy, count): + """The failures a run of the tests reports, or None, with the run's + output, when it reports none: the code did not build, or the run did + not finish. A run that dies writes no record, so the old one is + removed first rather than read again.""" + record = copy / "tests" / "TEST-RESULTS.md" + record.unlink(missing_ok=True) + run = subprocess.run([sys.executable, "tests/run_tests.py", "--count", str(count), + "--no-oracles"], + cwd=copy, capture_output=True, text=True) + out = run.stdout + run.stderr + if not record.exists(): + return None, f"no record written, exit status {run.returncode}\n{out}" + m = re.search(r"^- Failures: (\d+)(.*)$", record.read_text(encoding="utf-8"), re.M) + if m is None or m.group(2).strip(): + return None, out # the run gave up, for example the bug did not build: not a catch + n = int(m.group(1)) + if run.returncode != (1 if n else 0): + return None, f"{n} failures but exit status {run.returncode}\n{out}" + return n, None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--count", type=int, default=1000) + args = ap.parse_args() + rows, missed = [], 0 + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "path" + shutil.copytree(ROOT, copy, symlinks=True, + ignore=shutil.ignore_patterns("PLANTED-BUGS.md", "__pycache__")) + n, err = failures(copy, args.count) + rows.append(("none", n)) + if n != 0: + missed += 1 + print(f"none: {n} failures" + (" (did not build or did not finish)" if err else ""), + flush=True) + for what, line, bug, *where in BUGS: + name = where[0] if where else CODE + code = copy / name + original = code.read_text(encoding="utf-8") + if original.count(line) != 1: + sys.exit(f"{name} no longer holds, once, the text for {what!r}: {line}") + code.write_text(original.replace(line, bug), encoding="utf-8") + n, err = failures(copy, args.count) + code.write_text(original, encoding="utf-8") + rows.append((what, n)) + if not n: + missed += 1 + print(f"{what}: {n} failures" + (" (did not build or did not finish)" if err else ""), + flush=True) + if err: + print(err[-2000:], flush=True) + lines = ["# Planted bugs", "", + "Written by `tests/plant_bugs.py`. Each bug is a change of one line, or of two, " + f"to `{CODE}` unless it names `Main.lean`, tested with " + f"`tests/run_tests.py --count {args.count} --no-oracles`, which builds only the " + "executable and compares it with the model alone. Every bug must give failures, " + "and the unchanged code none. A bug that does not build, or a run that gives up " + "or dies, counts as missed.", "", + "| Planted bug | Failures |", "|---|---|"] + lines += [f"| {what} | {'did not build or did not finish' if n is None else n} |" + for what, n in rows] + lines += ["", f"Bugs caught: {sum(1 for what, n in rows[1:] if n)} of {len(BUGS)}", ""] + RECORD.write_text("\n".join(lines), encoding="utf-8") + return 1 if missed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 0000000..e85d320 --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,785 @@ +#!/usr/bin/env python3 +"""Differential tests: the Lean `Path.clean` and `Path.join` against a +model of Go's rules and three implementations. + +The tool answers `clean` and `join` requests (see the header of +Main.lean). Its answers are checked two ways. + +1. Against the model, written from the documentation of Go's + `path.Clean` and `path.Join` and sharing nothing with the Lean + code. It has two parts, and they must agree on every path whose + cleaning is checked below: + + - `rules_clean` splits the path into its elements and applies the + four rules of the documentation, one at a time, until none + applies: drop an empty element (rule 1, and the slash at the end), + drop a `.` (rule 2), drop a `..` and the name before it (rule 3), + drop a `..` at the start of a rooted path (rule 4). The rule and + the place it applies at are picked at random, from a recorded + seed, so the order differs from path to path. It then writes the + elements back, after a slash if the path is rooted, and gives `.` + for nothing. + - `model_clean` reads the elements once, left to right, with a + stack, as Rob Pike's "Lexical File Names in Plan 9" does. It is + fast enough for long inputs, which `rules_clean` is not. + + `model_join` is the documentation of `Join`: the empty elements left + out, the rest joined by slashes and cleaned, and the empty string + when nothing is left. The model must also give every result in Go's + own tests of Clean and Join (`cleantests` and `jointests` in + src/path/path_test.go). Every answer of the tool must equal the + model's. + +2. Against three oracles, on every request each can answer: + + - Go: `path.Clean` and `path.Join` of the Go standard library, the + installed toolchain, through tests/oracles/go/main.go; + - Python: `posixpath.normpath`, through + tests/oracles/python/oracle.py; `posixpath.join` keeps only the + last rooted argument, so it is not asked `join`; + - Node: `path.posix.normalize` and `path.posix.join`, through + tests/oracles/node/oracle.js. + + An oracle that cannot be built or started is left out, and the + record says so on its `- Oracles left out:` line. A disagreement is + allowed only for a named reason that the model confirms, with the + oracle's own rule: + + - Python keeps two slashes at the start of a path, and only two, as + POSIX allows ("A pathname that begins with two successive slashes + may be interpreted in an implementation-defined manner"); the + model confirms it when Python gives `/` and then the model's + result. + - Node keeps a slash at the end of a path; the model confirms it + when the path ends in a slash and Node gives the model's result + and then a slash. + - Node's `join` gives `.`, not the empty string, when every element + is empty or there are none; and keeps a slash at the end as + `normalize` does. + + Any other disagreement is a failure. Each named one is counted. + +Inputs come from a recorded seed, after fixed sets: + +- Go's `cleantests` (each path, and each result) and `jointests`, and + the paths of Pike's paper; +- every path of up to 10 characters over `a`, `.` and `/`, and of up + to 7 over `a`, `b`, `.` and `/`; +- random paths of up to 80 characters, built from `/`, `.`, `..`, + names, and characters that are not ASCII, a space, a tab, a + backslash, NUL, CR and LF; +- random joins of zero to five elements, among them empty ones; +- long inputs of about a million characters: names, `..`, `.`, + slashes, and deep paths that `..` takes back to the start; the time + they take is recorded; +- malformed requests, which the tool must answer with `error`, among + them hex that is not UTF-8; +- the command-line modes `clean` and `join`, and wrong arguments: what + each run prints on standard output and its exit status. + +Every seventh request ends in CR LF. + + python3 tests/run_tests.py [--seed N] [--count N] [--no-oracles] + +`--count` is the number of random paths; half as many random joins are +made. `--no-oracles` leaves every oracle out and compares the tool with +the model alone, as tests/plant_bugs.py does. The script builds the +executable and the oracles first, writes tests/TEST-RESULTS.md, and +exits non-zero if any check fails. +""" + +import argparse +import itertools +import os +import random +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +EXE = ROOT / ".lake" / "build" / "bin" / "path" +TESTS = ROOT / "tests" +ORACLES = TESTS / "oracles" +GO_SRC = ORACLES / "go" +GO_EXE = ROOT / ".lake" / "oracle-go" / "go-oracle" +PY_ORACLE = ORACLES / "python" / "oracle.py" +NODE_ORACLE = ORACLES / "node" / "oracle.js" +RESULTS = TESTS / "TEST-RESULTS.md" +TIMEOUT = 3600 +BUILD_TIMEOUT = 3600 +# The requests are split among this many `path serve` processes. +JOBS = max(1, min(4, os.cpu_count() or 1)) + +# --------------------------------------------------------------------- +# The model. + +DOT, DOTDOT = ".", ".." +CRLF = "\r\n" + + +def is_name(e): + return e not in ("", DOT, DOTDOT) + + +def render(rooted, es): + """The path with the elements `es`: a slash first when it is + rooted, `.` when it is empty and not rooted.""" + s = "/".join(es) + if rooted: + return "/" + s + return s or "." + + +def rules_clean(p, rng): + """Go's four rules, applied one at a time, at a place picked by + `rng`, until none applies.""" + rooted = p.startswith("/") + es = p.split("/") + while True: + moves = [] + for i, e in enumerate(es): + if e == "": + moves.append(("empty", i)) # rule 1 + elif e == DOT: + moves.append(("dot", i)) # rule 2 + elif e == DOTDOT and i > 0 and is_name(es[i - 1]): + moves.append(("parent", i)) # rule 3 + if rooted and es and es[0] == DOTDOT: + moves.append(("root", 0)) # rule 4 + if not moves: + return render(rooted, es) + rule, i = rng.choice(moves) + if rule == "parent": + del es[i - 1:i + 1] + else: + del es[i] + + +def model_clean(p): + """The elements read once, with a stack.""" + rooted = p.startswith("/") + stack = [] + for e in p.split("/"): + if e in ("", DOT): + continue + if e == DOTDOT: + if stack and stack[-1] != DOTDOT: + stack.pop() + elif not rooted: + stack.append(DOTDOT) + continue + stack.append(e) + return render(rooted, stack) + + +def model_join(es): + kept = [e for e in es if e != ""] + if not kept: + return "" + return model_clean("/".join(kept)) + + +# Go's tests: src/path/path_test.go, go1.24.7. +GO_CLEAN = [ + ("", "."), ("abc", "abc"), ("abc/def", "abc/def"), ("a/b/c", "a/b/c"), (".", "."), + ("..", ".."), ("../..", "../.."), ("../../abc", "../../abc"), ("/abc", "/abc"), ("/", "/"), + ("abc/", "abc"), ("abc/def/", "abc/def"), ("a/b/c/", "a/b/c"), ("./", "."), ("../", ".."), + ("../../", "../.."), ("/abc/", "/abc"), + ("abc//def//ghi", "abc/def/ghi"), ("//abc", "/abc"), ("///abc", "/abc"), + ("//abc//", "/abc"), ("abc//", "abc"), + ("abc/./def", "abc/def"), ("/./abc/def", "/abc/def"), ("abc/.", "abc"), + ("abc/def/ghi/../jkl", "abc/def/jkl"), ("abc/def/../ghi/../jkl", "abc/jkl"), + ("abc/def/..", "abc"), ("abc/def/../..", "."), ("/abc/def/../..", "/"), + ("abc/def/../../..", ".."), ("/abc/def/../../..", "/"), + ("abc/def/../../../ghi/jkl/../../../mno", "../../mno"), + ("abc/./../def", "def"), ("abc//./../def", "def"), ("abc/../../././../def", "../../def"), +] +GO_JOIN = [ + ([], ""), ([""], ""), (["a"], "a"), (["a", "b"], "a/b"), (["a", ""], "a"), (["", "b"], "b"), + (["/", "a"], "/a"), (["/", ""], "/"), (["a/", "b"], "a/b"), (["a/", ""], "a"), (["", ""], ""), +] +# Paths from Rob Pike, "Lexical File Names in Plan 9, or Getting Dot-Dot +# Right", and a few more at the edges of the rules. +MORE_PATHS = ["/usr/rob/..", "/usr/rob/../..", "/usr/rob/../../..", "../../../a", "a/../../b", + "/../a/..", "./.././a", "a/./b/./c/./", "...", "a/.../b", ".a/..b/a.", "a/..b/..", + "....//....", "/./", "//", "///", "//..", "//../a", "/a//../b", " ", " /..", "\\..", + "a\\..\\b", "é/../ü", "日本/../語", "\x00/..", "a\nb/../c", "a\r/..", "\t/./"] + +# --------------------------------------------------------------------- +# Requests. + + +def hx(s): + return s.encode("utf-8").hex() or "-" + + +def request(op, args): + return " ".join([op] + [hx(a) for a in args]) + + +def model(op, args): + if op == "clean": + return "OK " + hx(model_clean(args[0])) + return "OK " + hx(model_join(args)) + + +def model_line(raw): + """What the tool must answer to a request line, malformed or not.""" + line = raw[:-1] if raw.endswith("\n") else raw + if line.endswith("\r") and raw.endswith("\n"): + line = line[:-1] + fs = line.split(" ") + + def field(f): + if f == "-": + return "" + if not f or len(f) % 2 or any(c not in "0123456789abcdef" for c in f): + return None + try: + return bytes.fromhex(f).decode("utf-8") + except ValueError: + return None + + if fs[0] == "clean" and len(fs) == 2: + p = field(fs[1]) + return "error" if p is None else model("clean", [p]) + if fs[0] == "join": + es = [field(f) for f in fs[1:]] + return "error" if any(e is None for e in es) else model("join", es) + return "error" + + +MALFORMED = [ + "", "clean", "clean 61 62", "clean 6", "clean 4A", "clean 0x61", "clean zz", "clean ", + "clean 61", "Clean 61", "CLEAN 61", "clean\t61", "clean 61 ", " clean 61", "clean --", + # not UTF-8: a lone continuation octet, an overlong slash, a surrogate, + # past U+10FFFF, a cut sequence + "clean ff", "clean 80", "clean c0af", "clean eda080", "clean f4908080", "clean e282", + "join 6", "join ff", "join 61 ", "join 61", "join 61 zz", "join -- 61", "joins 61", + "normalize 61", "clean2 61", "x", +] +# Last lines with no LF, with CR LF, with a lone CR: (request, answer). +AT_EOF = [("clean 2f2e2e", "OK 2f"), ("clean 612f2e2e\r\n", "OK 2e"), + ("join 61 62", "OK 612f62"), ("clean 61\r", "error")] + +# --------------------------------------------------------------------- +# Inputs: (label, op, args). + +ATOMS = ["/", "/", "/", ".", ".", "..", "a", "b", "abc", "x.y", "...", "é", "日本", "😀", " ", + "\t", "\\", "\x00", "\n", "\r", "//", "./", "../"] + + +def random_path(rng, n): + return "".join(rng.choice(ATOMS) for _ in range(rng.randrange(n + 1)))[:n] + + +def exhaustive(): + for n in range(11): + for t in itertools.product("a./", repeat=n): + yield ("every path over a . / up to 10", "clean", ["".join(t)]) + for n in range(8): + for t in itertools.product("ab./", repeat=n): + if "b" in t: + yield ("every path over a b . / up to 7", "clean", ["".join(t)]) + + +def long_inputs(): + m = 1_000_000 + return [ + ("long", "clean", ["a/" * (m // 2)]), + ("long", "clean", ["x" * m]), + ("long", "clean", ["/" * m]), + ("long", "clean", ["./" * (m // 2)]), + ("long", "clean", ["../" * (m // 3)]), + ("long", "clean", ["/" + "../" * (m // 3)]), + ("long", "clean", ["/" + "a/" * (m // 4) + "../" * (m // 6)]), + ("long", "clean", ["a/" * (m // 4) + "../" * (m // 4)]), + ("long", "clean", ["a/" * (m // 4) + "../" * (m // 3)]), + ("long", "clean", ["a/b/../" * (m // 7)]), + ("long", "clean", ["abcdefghij/" * (m // 22) + "../" * (m // 66)]), + ("long", "join", ["a/" * (m // 4), "../" * (m // 6), "b" * 1000]), + ] + + +def inputs(seed, count): + rng = random.Random(seed) + for p, r in GO_CLEAN: + yield ("Go cleantests", "clean", [p]) + yield ("Go cleantests, the result", "clean", [r]) + for es, _ in GO_JOIN: + yield ("Go jointests", "join", es) + yield ("Go jointests", "join", ["", "/etc"]) + for p in MORE_PATHS: + yield ("Pike, and edges", "clean", [p]) + yield from exhaustive() + for _ in range(count): + yield ("random path", "clean", [random_path(rng, rng.choice([8, 20, 80]))]) + for _ in range(count // 2): + k = rng.randrange(6) + es = [random_path(rng, rng.choice([0, 3, 8, 20])) for _ in range(k)] + yield ("random join", "join", es) + yield from long_inputs() + + +# --------------------------------------------------------------------- +# Running a program over request lines. + +def run_lines(cmd, lines, timeout=TIMEOUT, cwd=None, env=None): + """Send `lines` to `cmd` and return its answer lines and a note on + how the run failed, or None.""" + raw = "".join(lines).encode("utf-8") + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=cwd, env=env) + out, err = bytearray(), bytearray() + + def feed(): + try: + proc.stdin.write(raw) + proc.stdin.close() + except (BrokenPipeError, ValueError): + pass + + def drain(stream, buf): + while chunk := stream.read1(1 << 16): + buf += chunk + + threads = [threading.Thread(target=feed), + threading.Thread(target=drain, args=(proc.stdout, out)), + threading.Thread(target=drain, args=(proc.stderr, err))] + for t in threads: + t.start() + problem = None + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + problem = f"no end after {timeout} seconds" + for t in threads: + t.join() + proc.wait() + if proc.returncode != 0 and not problem: + tail = " / ".join(err.decode("utf-8", "backslashreplace").splitlines()[-3:]) + problem = f"exit status {proc.returncode}: {tail}" + text = bytes(out).decode("utf-8", "backslashreplace") + if text and not text.endswith("\n"): + problem = problem or "the last answer has no LF" + answers = text.split("\n") + if answers and answers[-1] == "": + answers.pop() + return answers, problem + + +def run_parallel(cmd, lines, jobs): + """`run_lines` over `jobs` processes, each given a run of the lines + in order; the answers in order, and the problems.""" + size = -(-len(lines) // jobs) if lines else 0 + chunks = [lines[k:k + size] for k in range(0, len(lines), size)] if size else [[]] + results = [None] * len(chunks) + + def one(k): + results[k] = run_lines(cmd, chunks[k]) + + threads = [threading.Thread(target=one, args=(k,)) for k in range(len(chunks))] + for t in threads: + t.start() + for t in threads: + t.join() + answers, problems = [], [] + for chunk, (got, problem) in zip(chunks, results): + if problem: + problems.append(problem) + if len(got) != len(chunk): + problems.append(f"{len(got)} answers for {len(chunk)} requests") + got = (got + [""] * len(chunk))[:len(chunk)] + answers += got + return answers, problems + + +# --------------------------------------------------------------------- +# The oracles. Each setup returns an object with `name`, `version` and +# `answers(lines)`, or raises OracleUnavailable. + +class OracleUnavailable(Exception): + pass + + +class ProcessOracle: + def __init__(self, name, version, cmd): + self.name, self.version, self.cmd = name, version, cmd + + def answers(self, lines): + got, problem = run_lines(self.cmd, lines) + if problem or len(got) != len(lines): + raise RuntimeError(f"{self.name} oracle: " + f"{problem or f'{len(got)} answers for {len(lines)} requests'}") + return got + + +def first_line(cmd, cwd=None): + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=300, cwd=cwd) + return (out.stdout or out.stderr).strip().splitlines()[0] + except (OSError, subprocess.SubprocessError, IndexError): + return "unknown" + + +def setup_go(): + GO_EXE.parent.mkdir(parents=True, exist_ok=True) + env = dict(os.environ, GOFLAGS="-mod=readonly", CGO_ENABLED="0", GOTOOLCHAIN="local") + try: + subprocess.run(["go", "build", "-o", str(GO_EXE), "."], cwd=GO_SRC, check=True, env=env, + capture_output=True, timeout=BUILD_TIMEOUT) + except (OSError, subprocess.SubprocessError) as e: + raise OracleUnavailable(f"`go build` did not build it: {e}") + return ProcessOracle("Go", f"`path.Clean` and `path.Join` of the standard library of " + f"{first_line(['go', 'version'])}", [str(GO_EXE)]) + + +def setup_python(): + version = first_line([sys.executable, "-I", "--version"]) + return ProcessOracle("Python", f"`posixpath.normpath` of {version}", + [sys.executable, "-I", str(PY_ORACLE)]) + + +def setup_node(): + version = first_line(["node", "--version"]) + if version == "unknown": + raise OracleUnavailable("`node` is not installed") + return ProcessOracle("Node", f"`path.posix.normalize` and `path.posix.join` of Node " + f"{version}", ["node", str(NODE_ORACLE)]) + + +ORACLE_SETUPS = [("Go", setup_go), ("Python", setup_python), ("Node", setup_node)] + +TWO_SLASHES = "Python keeps two slashes at the start of a path, as POSIX allows" +TRAILING = "Node keeps a slash at the end of a path" +JOIN_EMPTY = "Node's `join` gives `.` when every element is empty, or there are none" + + +def difference_reason(oracle, op, args, want, theirs): + """The named reason, confirmed by the model, for an oracle that + answers `theirs` where the model answers `want`; or None.""" + if oracle == "Python" and op == "clean": + p = args[0] + if p.startswith("//") and not p.startswith("///") and \ + theirs == "OK " + hx("/" + model_clean(p)): + return TWO_SLASHES + if oracle == "Node": + if op == "clean": + p = args[0] + if p.endswith("/") and theirs == "OK " + hx(model_clean(p) + "/"): + return TRAILING + if op == "join": + kept = [e for e in args if e != ""] + if not kept and theirs == "OK " + hx("."): + return JOIN_EMPTY + joined = "/".join(kept) + if kept and joined.endswith("/") and \ + theirs == "OK " + hx(model_clean(joined) + "/"): + return TRAILING + " (in `join`)" + return None + + +# --------------------------------------------------------------------- +# The command line. + +def cli_cases(): + """Runs of the command-line modes: (arguments, what the run must + print on standard output, without the last LF, and its exit + status), from the model.""" + cases = [] + for p in ["", "a//b/../c/", "/../x", "..", "é/./ü/", "a b/ c"]: + cases.append((["clean", p], model_clean(p), 0)) + for es in [[], [""], ["a", "b"], ["", "/etc"], ["/", "..", "x/"], ["a", "", "", "b"]]: + cases.append((["join"] + es, model_join(es), 0)) + for args in [[], ["bogus"], ["clean"], ["clean", "a", "b"], ["serve", "x"], ["Clean", "a"]]: + cases.append((args, "", 2)) + return cases + + +def run_cli(cases): + right, failures = 0, [] + for args, out, status in cases: + try: + p = subprocess.run([str(EXE)] + args, capture_output=True, timeout=60) + got, code_ = p.stdout.decode("utf-8", "backslashreplace"), p.returncode + if got.endswith("\n"): + got = got[:-1] + except (OSError, subprocess.SubprocessError) as e: + got, code_ = str(e), None + if (got, code_) == (out, status): + right += 1 + else: + failures.append((f"path {' '.join(repr(a) for a in args)}", + f"{out!r}, exit {status}", f"{short(got, 160)!r}, exit {code_}")) + return right, failures + +# --------------------------------------------------------------------- + + +def lean_version(): + try: + out = subprocess.run(["lake", "env", "lean", "--version"], cwd=ROOT, + capture_output=True, text=True, errors="backslashreplace", + timeout=300) + first = out.stdout.strip().splitlines()[:1] + return first[0] if first else "unknown" + except (OSError, subprocess.TimeoutExpired): + return "unknown" + + +def count_arg(text): + n = int(text) + if n < 1: + raise argparse.ArgumentTypeError("the count must be at least 1") + return n + + +def write(lines): + RESULTS.write_text("\n".join(lines), encoding="utf-8") + + +def code(text): + """`text` as Markdown inline code, whatever backticks it holds.""" + return f"`` {text} ``" if "`" in text else f"`{text}`" + + +def short(text, n=200): + return text if len(text) <= n else text[:n] + f"... ({len(text)} characters)" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--seed", type=int, default=20261027) + ap.add_argument("--count", type=count_arg, default=40000) + ap.add_argument("--no-oracles", action="store_true", + help="compare with the model only, as tests/plant_bugs.py does") + args = ap.parse_args() + os.environ["PATH"] = f"{Path.home() / '.elan' / 'bin'}{os.pathsep}{os.environ.get('PATH', '')}" + + def give_up(why): + write(["# Test results", "", "- Oracle: none ran", f"- Seed: {args.seed}", "- Inputs: 0", + f"- Failures: 1 ({why})", ""]) + sys.exit(why) + + try: + build = subprocess.run(["lake", "build", "path"], cwd=ROOT, capture_output=True, + text=True, errors="backslashreplace", timeout=BUILD_TIMEOUT) + if build.returncode != 0: + print(build.stdout + build.stderr) + give_up("`lake build path` failed") + except (OSError, subprocess.SubprocessError) as e: + give_up(f"`lake build path` failed: {e}") + + failures = [] + + # The model against Go's tests. + go_right = 0 + for p, r in GO_CLEAN: + ok = model_clean(p) == r and model_clean(r) == r + go_right += ok + if not ok: + failures.append((f"the model on Go's cleantests {p!r}", repr(r), repr(model_clean(p)))) + for es, r in GO_JOIN: + ok = model_join(es) == r + go_right += ok + if not ok: + failures.append((f"the model on Go's jointests {es!r}", repr(r), repr(model_join(es)))) + + cli = cli_cases() + cli_right, cli_failures = run_cli(cli) + failures += cli_failures + + oracles, left_out = [], [] + for name, setup in ORACLE_SETUPS: + if args.no_oracles: + left_out.append((name, "--no-oracles")) + continue + try: + oracles.append(setup()) + except Exception as e: # any oracle that cannot start is left out, and recorded + left_out.append((name, str(e).splitlines()[0][:300] if str(e) else type(e).__name__)) + for name, why in left_out: + print(f"oracle left out: {name}: {why}") + + items = list(inputs(args.seed, args.count)) + long_items = [it for it in items if it[0] == "long"] + items = [it for it in items if it[0] != "long"] + requests = [request(op, a) for _, op, a in items] + lines = [r + ("\r\n" if i % 7 == 6 else "\n") for i, r in enumerate(requests)] + out, problems = run_parallel([str(EXE), "serve"], lines + [m + "\n" for m in MALFORMED], + JOBS) + for problem in problems: + failures.append(("(whole run)", "a clean run", problem)) + tool, malformed_out = out[:len(items)], out[len(items):] + malformed_right = sum(1 for a in malformed_out if a == "error") + for m, a in zip(MALFORMED, malformed_out): + if model_line(m + "\n") != "error": + failures.append((f"the model on {m!r}", "error", model_line(m + "\n"))) + if a != "error": + failures.append((f"request {m!r}", "error", a)) + eof_right = 0 + for req, want in AT_EOF: + if model_line(req) != want: + failures.append((f"the model on {req!r} at end of input", want, model_line(req))) + got, problem = run_lines([str(EXE), "serve"], [req], timeout=60) + if got == [want] and not problem: + eof_right += 1 + else: + failures.append((repr(req) + " (at end of input)", want, f"{got} {problem or ''}".strip())) + + # The long inputs, one process, timed. + long_lines = [request(op, a) + "\n" for _, op, a in long_items] + start = time.monotonic() + long_out, problem = run_lines([str(EXE), "serve"], long_lines) + long_secs = time.monotonic() - start + if problem or len(long_out) != len(long_lines): + failures.append(("(long inputs)", "a clean run", + problem or f"{len(long_out)} answers for {len(long_lines)} requests")) + long_out = (long_out + [""] * len(long_lines))[:len(long_lines)] + long_chars = sum(sum(len(x) for x in a) for _, _, a in long_items) + + # The two parts of the model against each other: every path of the + # exhaustive sets, Go's, Pike's, and the random paths. + rng = random.Random(args.seed + 1) + rules_checked = rules_agree = 0 + for _, op, a in items: + if op == "clean": + rules_checked += 1 + r, m = rules_clean(a[0], rng), model_clean(a[0]) + if r == m: + rules_agree += 1 + else: + failures.append((f"the model's two parts on {a[0]!r}", repr(r), repr(m))) + + # Against the model. + wants = [] + differ_model = 0 + for (label, op, a), got in zip(items, tool): + want = model(op, a) + wants.append(want) + if got != want: + differ_model += 1 + failures.append((short(request(op, a)), short(want, 160), short(got, 160))) + for (label, op, a), got in zip(long_items, long_out): + want = model(op, a) + if got != want: + differ_model += 1 + failures.append((short(request(op, a)), short(want, 160), short(got, 160))) + + # Against the oracles: the same lines, the long ones too. + all_items = items + long_items + all_lines = lines + long_lines + all_wants = wants + [model(op, a) for _, op, a in long_items] + oracle_out = {} + + def ask(o): + try: + oracle_out[o.name] = o.answers(all_lines) + except (RuntimeError, OSError) as e: + oracle_out[o.name] = e + + threads = [threading.Thread(target=ask, args=(o,)) for o in oracles] + for t in threads: + t.start() + for t in threads: + t.join() + for o in list(oracles): + if isinstance(oracle_out[o.name], Exception): + failures.append((f"({o.name} oracle run)", "a clean run", str(oracle_out[o.name])[:300])) + oracles.remove(o) + names = [o.name for o in oracles] + asked = {o: 0 for o in names} + agree = {o: 0 for o in names} + by_op = {o: {} for o in names} + named = {o: {} for o in names} + for i, (label, op, a) in enumerate(all_items): + want = all_wants[i] + for o in names: + theirs = oracle_out[o][i] + if theirs == "n/a": + continue + asked[o] += 1 + by_op[o][op] = by_op[o].get(op, 0) + 1 + if theirs == want: + agree[o] += 1 + continue + reason = difference_reason(o, op, a, want, theirs) + if reason is None: + failures.append((f"{o}: {short(request(op, a))}", short(want, 160), + short(theirs, 160))) + else: + n, example = named[o].get(reason, (0, None)) + named[o][reason] = (n + 1, example or request(op, a)) + + by_label = {} + for label, op, _ in items + long_items: + key = f"{label} ({op})" + by_label[key] = by_label.get(key, 0) + 1 + n_clean = sum(1 for _, op, _ in all_items if op == "clean") + n_join = len(all_items) - n_clean + + lines_out = [ + "# Test results", + "", + "Written by `tests/run_tests.py`. Rerun it to check these numbers.", + "", + f"- Lean: {lean_version()}", + "- Oracle: the model of Go's rules in `tests/run_tests.py`" + + "".join(f"; {o.name}: {o.version}" for o in oracles), + ] + lines_out += [f"- {o.name}: {o.version}" for o in oracles] + lines_out += [ + "- Oracles left out: " + ("; ".join(f"{n} ({why})" for n, why in left_out) or "none"), + f"- Seed: {args.seed}", + f"- Inputs: {len(all_items)}", + "- By kind: " + ", ".join(f"{k} {n}" for k, n in by_label.items()), + f"- Requests: `clean` {n_clean}, `join` {n_join}", + f"- The model on Go's `cleantests` and `jointests`: {go_right} of " + f"{len(GO_CLEAN) + len(GO_JOIN)}", + f"- The model's two parts, the rules applied in a random order and the stack, agree on " + f"{rules_agree} of {rules_checked} paths", + f"- Answers that differ from the model: {differ_model}", + f"- Long inputs: {len(long_items)}, {long_chars} characters in all, answered in " + f"{long_secs:.1f} s by one `path serve`", + "- Agreement with each oracle over the requests it answers: " + + (", ".join(f"{o} {agree[o]} of {asked[o]}" for o in names) or "no oracle ran"), + "- Requests each oracle answers, by operation: " + + ("; ".join(f"{o}: " + ", ".join(f"{op} {n}" for op, n in sorted(by_op[o].items())) + for o in names) or "no oracle ran"), + f"- Malformed requests answered `error`: {malformed_right} of {len(MALFORMED)}", + f"- Requests ending in CR LF: every seventh, {sum(1 for l in lines if l.endswith(CRLF))}", + f"- Last lines sent alone, with no LF, with CR LF, or with a lone CR, answered right: " + f"{eof_right} of {len(AT_EOF)}", + f"- Command-line modes (`clean`, `join`, and wrong arguments), printing and exiting as " + f"the model says: {cli_right} of {len(cli)}", + f"- Failures: {len(failures)}", + "", + ] + if any(named[o] for o in names): + lines_out += ["## Where the oracles differ", "", + "Each disagreement is counted under the reason the model confirms, with " + "the first input that showed it.", ""] + for o in names: + if not named[o]: + continue + lines_out += [f"### {o}", ""] + for reason, (n, example) in sorted(named[o].items(), key=lambda kv: -kv[1][0]): + lines_out.append(f"- {reason}: {n}; for example {code(short(example))}") + lines_out.append("") + if failures: + lines_out += ["## Failures", "", "| Input | Expected | Got |", "|---|---|---|"] + for what, want, got in failures[:200]: + lines_out.append(f"| {code(what)} | {code(want)} | {code(got)} |") + if len(failures) > 200: + lines_out.append(f"| ... | {len(failures) - 200} more | |") + lines_out.append("") + write(lines_out) + print(f"{len(all_items)} inputs, {len(failures)} failures") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) From eb3f5fb8e52b0eafddbf5272b3663205200cb6c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 00:23:22 +0000 Subject: [PATCH 2/3] Self-review: two more statements, rule 3 reading, honest page - Spec.step_shortens (every rule drops an element, so the rules end in any order) and Spec.rules_unique (every order ends at the same elements) back what the page said of the order; 12 statements now. - Step.parent and SPEC.md say why rule 3's "non-.. element" is read as a name: read literally, `abc/./../def` could end as `abc/def` or `def`. - Path.clean and SPEC.md no longer call the result the shortest path, which nothing proves. - README lists Go's code as tested, not proved, and says that the CLI replaces argument octets that are not UTF-8 with U+FFFD; the tests now run the CLI on such arguments. - The record gives the extra join case its own label. lake comparator --paranoid accepts all 12 statements. 167,263 inputs, 0 failures; 22 of 22 planted bugs caught. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AovXVgJBHGiN44TxQM4dcd --- Path/Basic.lean | 4 ++-- Proofs/Main.lean | 9 +++++++++ README.md | 9 ++++++--- SPEC.md | 10 ++++++---- Spec.lean | 13 +++++++++++++ Spec/Defs.lean | 7 ++++++- comparator.json | 3 ++- tests/PLANTED-BUGS.md | 8 ++++---- tests/TEST-RESULTS.md | 6 +++--- tests/run_tests.py | 19 ++++++++++++++----- 10 files changed, 65 insertions(+), 23 deletions(-) diff --git a/Path/Basic.lean b/Path/Basic.lean index 23b48d0..b549e72 100644 --- a/Path/Basic.lean +++ b/Path/Basic.lean @@ -74,8 +74,8 @@ def cleanChars (path : List Char) : List Char := -- Turn empty string into "." if w == 0 then ['.'] else out.reverse -/-- The shortest path equivalent to `path` by lexical processing alone, - as Go's `path.Clean` gives it. -/ +/-- The path cleaned by lexical processing alone, as Go's `path.Clean` + cleans it. -/ def clean (path : String) : String := String.ofList (cleanChars path.toList) diff --git a/Proofs/Main.lean b/Proofs/Main.lean index 7dee5db..5f9ca77 100644 --- a/Proofs/Main.lean +++ b/Proofs/Main.lean @@ -21,6 +21,15 @@ theorem rules_end (p : String) : Normal (Rooted p.toList) es' := ⟨elems p.toList, nf _ (elems p.toList), elems_splits _, steps_nf_self _ _, nf_normal _ _⟩ +theorem step_shortens (rooted : Bool) (es es' : List Element) (h : Step rooted es es') : + es'.length < es.length := by + cases h <;> simp <;> omega + +theorem rules_unique (rooted : Bool) (es e1 e2 : List Element) + (h1 : Steps rooted es e1) (n1 : Normal rooted e1) + (h2 : Steps rooted es e2) (n2 : Normal rooted e2) : e1 = e2 := by + rw [steps_normal_eq _ _ _ h1 n1, steps_normal_eq _ _ _ h2 n2] + theorem normal_iff (rooted : Bool) (es : List Element) : Normal rooted es ↔ ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ diff --git a/README.md b/README.md index 395413e..b2c3ed9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# path: lexical path cleaning (Go's path.Clean), proved +# path: lexical path cleaning by Go's rules, proved Programs clean a path before they use it: they drop `.`, let each `..` remove the name before it, and squeeze repeated slashes. A server that joins a path from a request to its own folder relies on that cleaning to keep the request inside the folder, and a mistake there lets a request read files outside it. Go's `path.Clean` gives the rules most often copied, from Rob Pike's "Lexical File Names in Plan 9". This library is those rules, and Go's `path.Join`, in Lean, proved to follow Go's documentation. @@ -6,7 +6,7 @@ It only reads the string. It never looks at a file system, so it says nothing ab Machine-checked proofs show: -- **Clean is what Go's four rules give.** Go's documentation says Clean applies four rules "iteratively until no further processing can be done". Applied in any order, the rules end at the same elements, and `clean` gives them, written back as a path. +- **Clean is what Go's four rules give.** Go's documentation says Clean applies four rules "iteratively until no further processing can be done". Applied in any order, the rules come to an end, and at the same elements, and `clean` gives them, written back as a path. - **The result is clean.** It has no empty element and no `.`, and `..` only at its start, and none if the path is rooted. Cleaning it again changes nothing. - **Join is what Go's documentation says**, and **joined paths stay inside.** A clean relative path with no `..`, joined to any folder, gives the folder's elements and then its own. Any path cleaned as a rooted path, `clean ("/" ++ u)`, and joined to a folder that is not the empty string, never leaves the folder. - **Go's own tests** of Clean and Join give Go's results. @@ -32,6 +32,8 @@ The statements are in `Spec.lean`. The rules they use are in `Spec/Defs.lean`, w |-------|---------| | Whatever order Go's rules are applied in to the elements of a path, once none applies, the path they leave is `clean p`. | `Spec.clean_rules` | | The rules can always be applied until none applies. | `Spec.rules_end` | +| Every rule drops an element, so in any order the rules come to an end. | `Spec.step_shortens` | +| In any order, the rules end at the same elements. | `Spec.rules_unique` | | No rule applies exactly when the elements are some `..`, none if the path is rooted, then names. | `Spec.normal_iff` | | `clean p` is such a path, and rooted exactly when `p` is. | `Spec.clean_names` | | Cleaning a clean path changes nothing. | `Spec.clean_idem` | @@ -42,11 +44,12 @@ The statements are in `Spec.lean`. The rules they use are in `Spec/Defs.lean`, w ## What is not proved +- **Go's code.** The proofs are against Go's documentation, written down in `Spec/Defs.lean`. That `clean` and `join` give what go1.24.7's `path.Clean` and `path.Join` give is tested (`tests/TEST-RESULTS.md`), not proved; only Go's own test tables are proved. - **The file system.** The rules are lexical. If `b/x` is a symbolic link, `b/x/..` on the file system need not be `b`, and a link inside a folder can point outside it. - **Two slashes at the start.** POSIX lets a system give `//` at the start of a path a meaning of its own. Go, and this library, read it as `/`; Python keeps it. That is a choice, recorded in `SPEC.md`. - **The empty folder.** `join ["", u]` leaves the empty element out, so `join ["", clean ("/" ++ u)]` is rooted. `join_root_inside` needs a folder that is not empty. - **Other path functions** of Go: `Split`, `Base`, `Dir`, `Ext`, `IsAbs` and `Match`. Windows paths are not covered. -- **Strings that are not UTF-8.** Lean strings are Unicode text, so a Go string holding other octets cannot be passed. +- **Strings that are not UTF-8.** Lean strings are Unicode text, so a Go string holding other octets cannot be passed to the library. On the command line, the Lean runtime replaces each run of such octets in an argument with U+FFFD before `path clean` and `path join` see it, so for such an argument the tool prints a path Go would not. `path serve` refuses them. - **Time and memory.** No proof bounds them. `clean` reads the path once, as Go's loop does, and removes each character it writes at most once; the tests clean paths of a million characters. - **Reading requests and printing answers in `Main.lean`.** The differential tests cover them. diff --git a/SPEC.md b/SPEC.md index 5f7e1c9..79b75b1 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,6 +1,6 @@ # Specification: path -Go's `path.Clean` (https://pkg.go.dev/path#Clean) gives "the shortest path name equivalent to path by purely lexical processing". It reads only the string: it never looks at a file system. This library does the same, and Go's `path.Join`, in Lean, on strings. +Go's documentation of `path.Clean` (https://pkg.go.dev/path#Clean) says it gives "the shortest path name equivalent to path by purely lexical processing". It reads only the string: it never looks at a file system. This library cleans paths by the rules of that documentation, and joins them as `path.Join` does, in Lean, on strings. That the result is the shortest such path is Go's description; it is not a statement here. A reviewer who did not write this specification approves it. The approval also covers `Spec.lean`, `Spec/Defs.lean`, and the other files that `bar.toml` lists. @@ -25,12 +25,12 @@ It adds: "The returned path ends in a slash only if it is the root `/`", and "If - **Rule 1** drops an empty element. On elements, rule 1 and the sentence on a slash at the end say the same thing. - **Rule 2** drops a `.` element. -- **Rule 3** drops a `..` element and the element before it, when that is a name: not empty, not `.` and not `..`. An empty element or a `.` before the `..` is dropped first, by rule 1 or 2. +- **Rule 3** drops a `..` element and the element before it, when that is a name: not empty, not `.` and not `..`. An empty element or a `.` before the `..` is dropped first, by rule 1 or 2. Go's words are "the non-`..` element that precedes it"; read literally, with an empty element or a `.` allowed there, the rules would end in more than one place. `abc/./../def` could end as `abc/def` or as `def`, and Go's own tests give `def`. - **Rule 4** drops a `..` at the start of a rooted path. `Spec.Steps` is the rules applied any number of times, in any order. `Spec.Normal` says that no rule applies. `Spec.render` writes elements back as a path: a slash first when it is rooted, a slash between each two elements, and `.` for no elements in a path that is not rooted. So the result ends in a slash only when it is the root `/`. -**Clean** is the path `render` writes for the elements the rules leave, in whatever order they are applied (`clean_rules`), and the rules can always be applied until none applies (`rules_end`). So the order does not matter: every order ends at the same elements. +**Clean** is the path `render` writes for the elements the rules leave, in whatever order they are applied (`clean_rules`), and the rules can always be applied until none applies (`rules_end`). Every rule drops an element, so in any order the rules come to an end (`step_shortens`), and every order ends at the same elements (`rules_unique`). No rule applies exactly when the elements are some `..`, none if the path is rooted, and then names (`normal_iff`). So the result of `clean` has no empty element and no `.`, and `..` only at its start, and none if it is rooted. It is rooted exactly when the path was (`clean_names`). Cleaning it again changes nothing (`clean_idem`). @@ -52,6 +52,8 @@ Each property is a theorem in `Spec.lean`, under the name given. |----------|---------| | Whatever order the rules are applied in, once none applies, `render` gives `clean p`. | `Spec.clean_rules` | | The rules can always be applied until none applies. | `Spec.rules_end` | +| Every rule drops an element, so in any order the rules come to an end. | `Spec.step_shortens` | +| In any order, the rules end at the same elements. | `Spec.rules_unique` | | No rule applies exactly when the elements are some `..` (none if rooted), then names. | `Spec.normal_iff` | | `clean p` is `render` of such elements, rooted when `p` is. | `Spec.clean_names` | | Cleaning a clean path changes nothing. | `Spec.clean_idem` | @@ -73,4 +75,4 @@ The functions take and give Lean strings, which are Unicode text in UTF-8. Go's - `path join ELEM...` prints the elements joined and cleaned, and an empty line for no elements or only empty ones. - `path serve` answers requests from the differential tests, one per line. The header comment of `Main.lean` gives the protocol. -Wrong arguments exit with 2. Every answer comes from the functions above. +Wrong arguments exit with 2. Every answer comes from the functions above. The Lean runtime reads the arguments of `clean` and `join` as UTF-8 and replaces each run of octets that are not UTF-8 with U+FFFD, so for such an argument the tool prints a path Go would not; `path serve` answers `error` for them. diff --git a/Spec.lean b/Spec.lean index 938597c..eb7ac85 100644 --- a/Spec.lean +++ b/Spec.lean @@ -27,6 +27,19 @@ theorem rules_end (p : String) : Normal (Rooted p.toList) es' := by sorry +/-- Every rule drops at least one element, so the rules, applied in any + order, always come to an end. -/ +theorem step_shortens (rooted : Bool) (es es' : List Element) (h : Step rooted es es') : + es'.length < es.length := by + sorry + +/-- Applied in any order, the rules end at the same elements: once none + applies, what is left does not depend on the order. -/ +theorem rules_unique (rooted : Bool) (es e1 e2 : List Element) + (h1 : Steps rooted es e1) (n1 : Normal rooted e1) + (h2 : Steps rooted es e2) (n2 : Normal rooted e2) : e1 = e2 := by + sorry + /-- No rule applies exactly when the elements are some `..`, none if the path is rooted, then names: elements that are not empty, `.` or `..`. -/ diff --git a/Spec/Defs.lean b/Spec/Defs.lean index c56e06c..91d9bd4 100644 --- a/Spec/Defs.lean +++ b/Spec/Defs.lean @@ -63,7 +63,12 @@ inductive Step (rooted : Bool) : List Element → List Element → Prop /-- Rule 2: drop a `.` element. -/ | dot (xs ys : List Element) : Step rooted (xs ++ dot :: ys) (xs ++ ys) /-- Rule 3: drop a `..` element and the element before it, when that - is a name: not empty, not `.` and not `..`. -/ + is a name: not empty, not `.` and not `..`. Go's words are "the + non-.. element that precedes it". Read literally, with an empty + element or a `.` allowed there, the rules would end in more than one + place: `abc/./../def` could end as `abc/def` or `def`, and Go's own + tests give `def`. An empty element or a `.` before the `..` is + dropped first, by rule 1 or 2. -/ | parent (xs : List Element) (e : Element) (ys : List Element) : e ≠ [] → e ≠ dot → e ≠ dotdot → Step rooted (xs ++ e :: dotdot :: ys) (xs ++ ys) diff --git a/comparator.json b/comparator.json index c147aad..4732978 100644 --- a/comparator.json +++ b/comparator.json @@ -2,7 +2,8 @@ "challenge_module": "Spec", "solution_module": "Proofs", "theorem_names": [ - "Spec.clean_rules", "Spec.rules_end", "Spec.normal_iff", "Spec.clean_names", + "Spec.clean_rules", "Spec.rules_end", "Spec.step_shortens", "Spec.rules_unique", + "Spec.normal_iff", "Spec.clean_names", "Spec.clean_idem", "Spec.join_eq", "Spec.join_inside", "Spec.join_root_inside", "Spec.go_clean_tests", "Spec.go_join_tests" ], diff --git a/tests/PLANTED-BUGS.md b/tests/PLANTED-BUGS.md index 09794da..9ad8865 100644 --- a/tests/PLANTED-BUGS.md +++ b/tests/PLANTED-BUGS.md @@ -11,12 +11,12 @@ Written by `tests/plant_bugs.py`. Each bug is a change of one line, or of two, t | a `..` that cannot backtrack is kept in a rooted path, not in one that is not | 6537 | | `dotdot` does not move past a `..` that is kept | 289 | | a kept `..` gets a slash before it even at the start | 4426 | -| a `.` at the end is kept | 10537 | +| a `.` at the end is kept | 10538 | | a `..` at the end is dropped | 3410 | | a `.` before a slash is copied as a name | 29229 | | in a rooted path, the first name gets a slash after the root | 32251 | -| a name runs on past the slash after it | 68609 | -| the length is not counted while a name is copied | 3347 | +| a name runs on past the slash after it | 68611 | +| the length is not counted while a name is copied | 3348 | | the root's `dotdot` is 0, so a `..` can remove the root | 1653 | | the root alone is not rooted | 15 | | an empty result is the empty string, not `.` | 2320 | @@ -25,7 +25,7 @@ Written by `tests/plant_bugs.py`. Each bug is a change of one line, or of two, t | Main.lean: `-` is read as the path `-` | 357 | | Main.lean: `clean` takes more than one field | 2 | | Main.lean: a CR before the LF is kept | 15536 | -| Main.lean: `path clean` prints the path as given | 4 | +| Main.lean: `path clean` prints the path as given | 6 | | Main.lean: wrong arguments exit with 0 | 6 | Bugs caught: 22 of 22 diff --git a/tests/TEST-RESULTS.md b/tests/TEST-RESULTS.md index 65fb798..ead416a 100644 --- a/tests/TEST-RESULTS.md +++ b/tests/TEST-RESULTS.md @@ -10,18 +10,18 @@ Written by `tests/run_tests.py`. Rerun it to check these numbers. - Oracles left out: none - Seed: 20261027 - Inputs: 167263 -- By kind: Go cleantests (clean) 36, Go cleantests, the result (clean) 36, Go jointests (join) 12, Pike, and edges (clean) 29, every path over a . / up to 10 (clean) 88573, every path over a b . / up to 7 (clean) 18565, random path (clean) 40000, random join (join) 20000, long (clean) 11, long (join) 1 +- By kind: Go cleantests (clean) 36, Go cleantests, the result (clean) 36, Go jointests (join) 11, an empty first element, a rooted second (join) 1, Pike, and edges (clean) 29, every path over a . / up to 10 (clean) 88573, every path over a b . / up to 7 (clean) 18565, random path (clean) 40000, random join (join) 20000, long (clean) 11, long (join) 1 - Requests: `clean` 147250, `join` 20013 - The model on Go's `cleantests` and `jointests`: 47 of 47 - The model's two parts, the rules applied in a random order and the stack, agree on 147239 of 147239 paths - Answers that differ from the model: 0 -- Long inputs: 12, 12296441 characters in all, answered in 3.3 s by one `path serve` +- Long inputs: 12, 12296441 characters in all, answered in 2.8 s by one `path serve` - Agreement with each oracle over the requests it answers: Go 167263 of 167263, Python 138020 of 147250, Node 117446 of 167263 - Requests each oracle answers, by operation: Go: clean 147250, join 20013; Python: clean 147250; Node: clean 147250, join 20013 - Malformed requests answered `error`: 31 of 31 - Requests ending in CR LF: every seventh, 23893 - Last lines sent alone, with no LF, with CR LF, or with a lone CR, answered right: 4 of 4 -- Command-line modes (`clean`, `join`, and wrong arguments), printing and exiting as the model says: 18 of 18 +- Command-line modes (`clean`, `join`, and wrong arguments), printing and exiting as the model says: 21 of 21 - Failures: 0 ## Where the oracles differ diff --git a/tests/run_tests.py b/tests/run_tests.py index e85d320..ecd169c 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -61,8 +61,9 @@ Inputs come from a recorded seed, after fixed sets: -- Go's `cleantests` (each path, and each result) and `jointests`, and - the paths of Pike's paper; +- Go's `cleantests` (each path, and each result) and `jointests`, a + join of an empty element and a rooted one, and the paths of Pike's + paper; - every path of up to 10 characters over `a`, `.` and `/`, and of up to 7 over `a`, `b`, `.` and `/`; - random paths of up to 80 characters, built from `/`, `.`, `..`, @@ -74,8 +75,10 @@ they take is recorded; - malformed requests, which the tool must answer with `error`, among them hex that is not UTF-8; -- the command-line modes `clean` and `join`, and wrong arguments: what - each run prints on standard output and its exit status. +- the command-line modes `clean` and `join`, among them arguments that + are not UTF-8, which the Lean runtime replaces with U+FFFD, and wrong + arguments: what each run prints on standard output and its exit + status. Every seventh request ends in CR LF. @@ -310,7 +313,7 @@ def inputs(seed, count): yield ("Go cleantests, the result", "clean", [r]) for es, _ in GO_JOIN: yield ("Go jointests", "join", es) - yield ("Go jointests", "join", ["", "/etc"]) + yield ("an empty first element, a rooted second", "join", ["", "/etc"]) for p in MORE_PATHS: yield ("Pike, and edges", "clean", [p]) yield from exhaustive() @@ -494,6 +497,12 @@ def cli_cases(): cases.append((["clean", p], model_clean(p), 0)) for es in [[], [""], ["a", "b"], ["", "/etc"], ["/", "..", "x/"], ["a", "", "", "b"]]: cases.append((["join"] + es, model_join(es), 0)) + # Arguments that are not UTF-8: the Lean runtime replaces each run of + # such octets with U+FFFD before `main` sees them, as Python's + # "replace" does for these. + for raw in [b"\xc3/x/.", b"a\xff/../b\xff"]: + cases.append((["clean", raw], model_clean(raw.decode("utf-8", "replace")), 0)) + cases.append((["join", b"\xff", "b"], model_join(["\ufffd", "b"]), 0)) for args in [[], ["bogus"], ["clean"], ["clean", "a", "b"], ["serve", "x"], ["Clean", "a"]]: cases.append((args, "", 2)) return cases From 7cb8358a4c1be9229ec5984fb5cdffe4f0fd628e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 00:31:48 +0000 Subject: [PATCH 3/3] Statements about the code: every approved statement names Path.clean The bar check asks every approved statement to mention the code under proof, and CI failed item 3 on rules_end, normal_iff, step_shortens and rules_unique, which spoke of the rules alone. They are restated: - clean_rules: the elements the rules leave are those `clean p` names (`Names`: rendered as `clean p`, no rule applies, no slash inside); - rules_end: the rules can always be applied until none applies, and leave the elements of `clean p`; - clean_names: `clean p` names exactly one list of elements, so every order of the rules ends at the same elements (was rules_unique); - clean_shape: those elements are some `..`, none if rooted, then names (was normal_iff). That the rules always end is read off `Spec.Step` on the page, not claimed as a statement. Proofs/Unique.lean proves that a path splits one way, that `Names` fixes its elements, and that the rules only drop elements. lake comparator --paranoid accepts all 10 statements, and bar/check.py passes item 3. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AovXVgJBHGiN44TxQM4dcd --- Proofs.lean | 1 + Proofs/Main.lean | 39 +++++------ Proofs/Unique.lean | 166 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 12 ++-- SPEC.md | 14 ++-- Spec.lean | 43 +++++------- comparator.json | 3 +- 7 files changed, 213 insertions(+), 65 deletions(-) create mode 100644 Proofs/Unique.lean diff --git a/Proofs.lean b/Proofs.lean index b9f0b73..05f8155 100644 --- a/Proofs.lean +++ b/Proofs.lean @@ -2,4 +2,5 @@ import Proofs.Rules import Proofs.Loop import Proofs.Clean import Proofs.Join +import Proofs.Unique import Proofs.Main diff --git a/Proofs/Main.lean b/Proofs/Main.lean index 5f9ca77..f14d7ea 100644 --- a/Proofs/Main.lean +++ b/Proofs/Main.lean @@ -4,7 +4,7 @@ here is exactly the one in `Spec.lean`. -/ -import Proofs.Join +import Proofs.Unique namespace Spec @@ -13,31 +13,26 @@ open Spec.Proof theorem clean_rules (p : String) (es es' : List Element) (hp : Splits p.toList es) (hs : Steps (Rooted p.toList) es es') (hn : Normal (Rooted p.toList) es') : - (Path.clean p).toList = render (Rooted p.toList) es' := by + Names (Path.clean p) (Rooted p.toList) es' := by + refine ⟨?_, hn, fun e he => hp.2.2 e (steps_subset _ _ _ hs e he)⟩ rw [Path.clean, String.toList_ofList, cleanChars_eq _ _ hp, ← steps_normal_eq _ _ _ hs hn] theorem rules_end (p : String) : ∃ es es', Splits p.toList es ∧ Steps (Rooted p.toList) es es' ∧ - Normal (Rooted p.toList) es' := - ⟨elems p.toList, nf _ (elems p.toList), elems_splits _, steps_nf_self _ _, nf_normal _ _⟩ - -theorem step_shortens (rooted : Bool) (es es' : List Element) (h : Step rooted es es') : - es'.length < es.length := by - cases h <;> simp <;> omega - -theorem rules_unique (rooted : Bool) (es e1 e2 : List Element) - (h1 : Steps rooted es e1) (n1 : Normal rooted e1) - (h2 : Steps rooted es e2) (n2 : Normal rooted e2) : e1 = e2 := by - rw [steps_normal_eq _ _ _ h1 n1, steps_normal_eq _ _ _ h2 n2] - -theorem normal_iff (rooted : Bool) (es : List Element) : - Normal rooted es ↔ - ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ - ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := - normal_iff_shape rooted es - -theorem clean_names (p : String) : ∃ es, Names (Path.clean p) (Rooted p.toList) es := - ⟨_, clean_names' p⟩ + Normal (Rooted p.toList) es' ∧ Names (Path.clean p) (Rooted p.toList) es' := + ⟨elems p.toList, nf _ (elems p.toList), elems_splits _, steps_nf_self _ _, nf_normal _ _, + clean_names' p⟩ + +theorem clean_names (p : String) : + ∃ es, Names (Path.clean p) (Rooted p.toList) es ∧ + ∀ es', Names (Path.clean p) (Rooted p.toList) es' → es' = es := + ⟨_, clean_names' p, fun _ h => names_unique _ _ _ _ h (clean_names' p)⟩ + +theorem clean_shape (p : String) (es : List Element) + (h : Names (Path.clean p) (Rooted p.toList) es) : + ∃ k names, es = List.replicate k dotdot ++ names ∧ (Rooted p.toList = true → k = 0) ∧ + ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := + shape_of_normal _ _ h.2.1 theorem clean_idem (p : String) : Path.clean (Path.clean p) = Path.clean p := by obtain ⟨h1, h2, h3⟩ := clean_names' p diff --git a/Proofs/Unique.lean b/Proofs/Unique.lean new file mode 100644 index 0000000..a1d12d6 --- /dev/null +++ b/Proofs/Unique.lean @@ -0,0 +1,166 @@ +import Proofs.Join + +/-! + A path splits into one list of elements, and a path names one list of + elements; the rules only drop elements, so they keep them free of + slashes. +-/ + +namespace Spec.Proof + +open Spec Path + +/-- Two slash-free elements, each followed by nothing or by a slash, are + the same element when the whole strings are. -/ +theorem slash_prefix (e f X Y : List Char) (he : '/' ∉ e) (hf : '/' ∉ f) + (hX : X = [] ∨ X.head? = some '/') (hY : Y = [] ∨ Y.head? = some '/') + (h : e ++ X = f ++ Y) : e = f ∧ X = Y := by + induction e generalizing f with + | nil => + cases f with + | nil => simpa using h + | cons c f => + exfalso + simp only [List.nil_append, List.cons_append] at h + subst h + rcases hX with hX | hX + · simp at hX + · simp only [List.head?_cons, Option.some.injEq] at hX + exact hf (by simp [hX]) + | cons c e ih => + cases f with + | nil => + exfalso + simp only [List.nil_append, List.cons_append] at h + subst h + rcases hY with hY | hY + · simp at hY + · simp only [List.head?_cons, Option.some.injEq] at hY + exact he (by simp [hY]) + | cons c' f => + simp only [List.cons_append, List.cons.injEq] at h + obtain ⟨rfl, h⟩ := h + obtain ⟨h1, h2⟩ := ih f (fun h => he (by simp [h])) (fun h => hf (by simp [h])) h + exact ⟨by rw [h1], h2⟩ + +/-- What follows the first element: nothing, or a slash and the rest. -/ +def after (E : List Element) : List Char := + match E with + | [] => [] + | _ :: _ => '/' :: List.intercalate ['/'] E + +theorem intercalate_cons (e : Element) (E : List Element) : + List.intercalate ['/'] (e :: E) = e ++ after E := by + cases E with + | nil => simp [after] + | cons e' E => simp [after, List.intercalate_cons_cons] + +theorem after_slash (E : List Element) : after E = [] ∨ (after E).head? = some '/' := by + cases E <;> simp [after] + +theorem splits_unique (p : List Char) (E F : List Element) (hE : Splits p E) + (hF : Splits p F) : E = F := by + induction E generalizing p F with + | nil => exact absurd rfl hE.1 + | cons e E ih => + obtain ⟨-, hp, hs⟩ := hE + obtain ⟨hF0, hq, ht⟩ := hF + obtain ⟨f, F, rfl⟩ := List.exists_cons_of_ne_nil hF0 + rw [intercalate_cons] at hp hq + obtain ⟨rfl, hrest⟩ := slash_prefix e f _ _ (hs e (by simp)) (ht f (by simp)) + (after_slash E) (after_slash F) (hp.symm.trans hq) + cases E with + | nil => + cases F with + | nil => rfl + | cons f' F => simp [after] at hrest + | cons e' E => + cases F with + | nil => simp [after] at hrest + | cons f' F => + simp only [after, List.cons.injEq, true_and] at hrest + rw [ih (List.intercalate ['/'] (e' :: E)) (f' :: F) + ⟨by simp, rfl, fun x hx => hs x (by simp [hx])⟩ + ⟨by simp, hrest, fun x hx => ht x (by simp [hx])⟩] + +/-- The elements `render` writes, split back: an empty element for the + root, and `.` for nothing. -/ +def rsplit (r : Bool) (E : List Element) : List Element := + if r then [] :: (if E = [] then [[]] else E) else if E = [] then [dot] else E + +theorem splits_rsplit (r : Bool) (E : List Element) (hE : ∀ e ∈ E, e ≠ [] ∧ '/' ∉ e) : + Splits (render r E) (rsplit r E) := by + have hEs : ∀ e ∈ E, '/' ∉ e := fun e he => (hE e he).2 + cases r with + | true => + by_cases h0 : E = [] + · subst h0; exact ⟨by simp [rsplit], by simp [render, rsplit], by simp [rsplit]⟩ + · refine ⟨by simp [rsplit], ?_, ?_⟩ + · obtain ⟨e, E', rfl⟩ := List.exists_cons_of_ne_nil h0 + simp [render, rsplit] + · simp only [rsplit, ite_true, h0, ite_false, List.mem_cons] + rintro x (rfl | hx) + · simp + · exact hEs x hx + | false => + by_cases h0 : E = [] + · subst h0; exact ⟨by simp [rsplit], by simp [render, rsplit, dot], by simp [rsplit, dot]⟩ + · exact ⟨by simp [rsplit, h0], by simp [render, rsplit, h0], by simp [rsplit, h0]; exact hEs⟩ + +theorem dot_not_normal (r : Bool) (E : List Element) (hn : Normal r E) : dot ∉ E := by + obtain ⟨k, names, rfl, -, hnames⟩ := shape_of_normal r E hn + intro h + rcases List.mem_append.mp h with h | h + · exact dotdot_ne_dot (List.mem_replicate.mp h).2.symm + · exact (hnames dot h).2.1 rfl + +/-- A path names one list of elements. -/ +theorem names_unique (s : String) (r : Bool) (e1 e2 : List Element) (h1 : Names s r e1) + (h2 : Names s r e2) : e1 = e2 := by + obtain ⟨h1s, h1n, h1f⟩ := h1 + obtain ⟨h2s, h2n, h2f⟩ := h2 + have hE1 := normal_elems r e1 h1n h1f + have hE2 := normal_elems r e2 h2n h2f + have hs := splits_rsplit r e1 hE1 + rw [← h1s, h2s] at hs + have heq := splits_unique _ _ _ (splits_rsplit r e2 hE2) hs + cases r with + | true => + simp only [rsplit, ite_true, List.cons.injEq, true_and] at heq + by_cases a : e1 = [] <;> by_cases b : e2 = [] <;> simp only [a, b, ite_true, ite_false] at heq + · rw [a, b] + · exact absurd (by rw [heq]; simp) (fun h => (hE2 [] h).1 rfl) + · exact absurd (by rw [← heq]; simp) (fun h => (hE1 [] h).1 rfl) + · exact heq.symm + | false => + simp only [rsplit, Bool.false_eq_true, ite_false] at heq + by_cases a : e1 = [] <;> by_cases b : e2 = [] <;> simp only [a, b, ite_true, ite_false] at heq + · rw [a, b] + · exact absurd (by rw [heq]; simp) (dot_not_normal false e2 h2n) + · exact absurd (by rw [← heq]; simp) (dot_not_normal false e1 h1n) + · exact heq.symm + +/-! ### The rules only drop elements -/ + +theorem step_subset (r : Bool) (es es' : List Element) (h : Step r es es') : + ∀ x ∈ es', x ∈ es := by + intro x hx + cases h with + | empty xs ys => + simp only [List.mem_append, List.mem_cons] at hx ⊢ + rcases hx with hx | hx <;> simp [hx] + | dot xs ys => + simp only [List.mem_append, List.mem_cons] at hx ⊢ + rcases hx with hx | hx <;> simp [hx] + | parent xs e ys => + simp only [List.mem_append, List.mem_cons] at hx ⊢ + rcases hx with hx | hx <;> simp [hx] + | root ys _ => exact List.mem_cons_of_mem _ hx + +theorem steps_subset (r : Bool) (es es' : List Element) (h : Steps r es es') : + ∀ x ∈ es', x ∈ es := by + induction h with + | refl => exact fun x hx => hx + | step h _ ih => exact fun x hx => step_subset r _ _ h x (ih x hx) + +end Spec.Proof diff --git a/README.md b/README.md index b2c3ed9..292bdcc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ It only reads the string. It never looks at a file system, so it says nothing ab Machine-checked proofs show: -- **Clean is what Go's four rules give.** Go's documentation says Clean applies four rules "iteratively until no further processing can be done". Applied in any order, the rules come to an end, and at the same elements, and `clean` gives them, written back as a path. +- **Clean is what Go's four rules give.** Go's documentation says Clean applies four rules "iteratively until no further processing can be done". Applied in any order until none applies, the rules leave the same elements, and `clean` gives them, written back as a path. - **The result is clean.** It has no empty element and no `.`, and `..` only at its start, and none if the path is rooted. Cleaning it again changes nothing. - **Join is what Go's documentation says**, and **joined paths stay inside.** A clean relative path with no `..`, joined to any folder, gives the folder's elements and then its own. Any path cleaned as a rooted path, `clean ("/" ++ u)`, and joined to a folder that is not the empty string, never leaves the folder. - **Go's own tests** of Clean and Join give Go's results. @@ -30,12 +30,10 @@ The statements are in `Spec.lean`. The rules they use are in `Spec/Defs.lean`, w | Claim | Theorem | |-------|---------| -| Whatever order Go's rules are applied in to the elements of a path, once none applies, the path they leave is `clean p`. | `Spec.clean_rules` | -| The rules can always be applied until none applies. | `Spec.rules_end` | -| Every rule drops an element, so in any order the rules come to an end. | `Spec.step_shortens` | -| In any order, the rules end at the same elements. | `Spec.rules_unique` | -| No rule applies exactly when the elements are some `..`, none if the path is rooted, then names. | `Spec.normal_iff` | -| `clean p` is such a path, and rooted exactly when `p` is. | `Spec.clean_names` | +| Whatever order Go's rules are applied in to the elements of a path, once none applies, the elements they leave are those of `clean p`. | `Spec.clean_rules` | +| The rules can always be applied until none applies, and they leave the elements of `clean p`. | `Spec.rules_end` | +| `clean p` names exactly one list of elements, as a path rooted exactly when `p` is; so in any order the rules end at the same elements. | `Spec.clean_names` | +| The elements of `clean p` are some `..`, none if `p` is rooted, then names: no empty element and no `.`. | `Spec.clean_shape` | | Cleaning a clean path changes nothing. | `Spec.clean_idem` | | `join` leaves out empty elements, joins the rest with slashes and cleans them, and gives the empty string when nothing is left. | `Spec.join_eq` | | A clean relative path with no `..`, joined to any path `b`, gives the elements of `clean b`, then its own. | `Spec.join_inside` | diff --git a/SPEC.md b/SPEC.md index 79b75b1..b36a78f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -30,9 +30,9 @@ It adds: "The returned path ends in a slash only if it is the root `/`", and "If `Spec.Steps` is the rules applied any number of times, in any order. `Spec.Normal` says that no rule applies. `Spec.render` writes elements back as a path: a slash first when it is rooted, a slash between each two elements, and `.` for no elements in a path that is not rooted. So the result ends in a slash only when it is the root `/`. -**Clean** is the path `render` writes for the elements the rules leave, in whatever order they are applied (`clean_rules`), and the rules can always be applied until none applies (`rules_end`). Every rule drops an element, so in any order the rules come to an end (`step_shortens`), and every order ends at the same elements (`rules_unique`). +**Clean** is the path `render` writes for the elements the rules leave, in whatever order they are applied (`clean_rules`), and the rules can always be applied until none applies (`rules_end`). `Spec.Names s r es` says that `s` is the path `render r` writes for `es`, that no rule applies to `es`, and that no element of `es` holds a slash; `clean p` names exactly one such list (`clean_names`). So every order of the rules ends at the same elements. Each rule drops one element or two, as `Spec.Step` shows, so the rules always come to an end. -No rule applies exactly when the elements are some `..`, none if the path is rooted, and then names (`normal_iff`). So the result of `clean` has no empty element and no `.`, and `..` only at its start, and none if it is rooted. It is rooted exactly when the path was (`clean_names`). Cleaning it again changes nothing (`clean_idem`). +The elements of `clean p` are some `..`, none if `p` is rooted, and then names (`clean_shape`). So the result of `clean` has no empty element and no `.`, and `..` only at its start, and none if it is rooted. It is rooted exactly when the path was (`clean_names`). Cleaning it again changes nothing (`clean_idem`). **Two slashes at the start.** POSIX says that "a pathname that begins with two successive slashes may be interpreted in an implementation-defined manner". Go reads them as one slash, and so does this library: `//abc` cleans to `/abc`, as in Go's own tests. Python's `posixpath.normpath` keeps exactly two. @@ -50,12 +50,10 @@ Each property is a theorem in `Spec.lean`, under the name given. | Property | Theorem | |----------|---------| -| Whatever order the rules are applied in, once none applies, `render` gives `clean p`. | `Spec.clean_rules` | -| The rules can always be applied until none applies. | `Spec.rules_end` | -| Every rule drops an element, so in any order the rules come to an end. | `Spec.step_shortens` | -| In any order, the rules end at the same elements. | `Spec.rules_unique` | -| No rule applies exactly when the elements are some `..` (none if rooted), then names. | `Spec.normal_iff` | -| `clean p` is `render` of such elements, rooted when `p` is. | `Spec.clean_names` | +| Whatever order the rules are applied in, once none applies, the elements left are those of `clean p`. | `Spec.clean_rules` | +| The rules can always be applied until none applies, and leave the elements of `clean p`. | `Spec.rules_end` | +| `clean p` names exactly one list of elements, rooted when `p` is. | `Spec.clean_names` | +| The elements of `clean p` are some `..` (none if rooted), then names. | `Spec.clean_shape` | | Cleaning a clean path changes nothing. | `Spec.clean_idem` | | `join` is what Go's documentation says. | `Spec.join_eq` | | A clean relative path with no `..`, joined to `b`, adds its elements to those of `clean b`. | `Spec.join_inside` | diff --git a/Spec.lean b/Spec.lean index eb7ac85..35dcdd1 100644 --- a/Spec.lean +++ b/Spec.lean @@ -13,45 +13,36 @@ import Spec.Defs namespace Spec /-- Whatever order the rules are applied in, once none applies, the - path with the elements left is `clean p`. -/ + elements left are those of `clean p`: it is the path `render` writes + for them. -/ theorem clean_rules (p : String) (es es' : List Element) (hp : Splits p.toList es) (hs : Steps (Rooted p.toList) es es') (hn : Normal (Rooted p.toList) es') : - (Path.clean p).toList = render (Rooted p.toList) es' := by + Names (Path.clean p) (Rooted p.toList) es' := by sorry /-- The rules can always be applied until none applies, so `clean_rules` says something about every path. -/ theorem rules_end (p : String) : ∃ es es', Splits p.toList es ∧ Steps (Rooted p.toList) es es' ∧ - Normal (Rooted p.toList) es' := by + Normal (Rooted p.toList) es' ∧ Names (Path.clean p) (Rooted p.toList) es' := by sorry -/-- Every rule drops at least one element, so the rules, applied in any - order, always come to an end. -/ -theorem step_shortens (rooted : Bool) (es es' : List Element) (h : Step rooted es es') : - es'.length < es.length := by +/-- `clean p` names exactly one list of elements, as a path rooted when + `p` is. With `clean_rules`, every order of the rules ends at the + same elements. -/ +theorem clean_names (p : String) : + ∃ es, Names (Path.clean p) (Rooted p.toList) es ∧ + ∀ es', Names (Path.clean p) (Rooted p.toList) es' → es' = es := by sorry -/-- Applied in any order, the rules end at the same elements: once none - applies, what is left does not depend on the order. -/ -theorem rules_unique (rooted : Bool) (es e1 e2 : List Element) - (h1 : Steps rooted es e1) (n1 : Normal rooted e1) - (h2 : Steps rooted es e2) (n2 : Normal rooted e2) : e1 = e2 := by - sorry - -/-- No rule applies exactly when the elements are some `..`, none if - the path is rooted, then names: elements that are not empty, `.` - or `..`. -/ -theorem normal_iff (rooted : Bool) (es : List Element) : - Normal rooted es ↔ - ∃ k names, es = List.replicate k dotdot ++ names ∧ (rooted = true → k = 0) ∧ - ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := by - sorry - -/-- So `clean p` has no empty element and no `.`, `..` only at the - start and none if `p` is rooted, and it is rooted when `p` is. -/ -theorem clean_names (p : String) : ∃ es, Names (Path.clean p) (Rooted p.toList) es := by +/-- The elements of `clean p` are some `..`, none if `p` is rooted, then + names: elements that are not empty, `.` or `..`. So `clean p` has + no empty element and no `.`, and `..` only at its start. -/ +theorem clean_shape (p : String) (es : List Element) + (h : Names (Path.clean p) (Rooted p.toList) es) : + ∃ k names, es = List.replicate k dotdot ++ names ∧ (Rooted p.toList = true → k = 0) ∧ + ∀ e ∈ names, e ≠ [] ∧ e ≠ dot ∧ e ≠ dotdot := by sorry /-- Cleaning a clean path changes nothing. -/ diff --git a/comparator.json b/comparator.json index 4732978..c43ab31 100644 --- a/comparator.json +++ b/comparator.json @@ -2,8 +2,7 @@ "challenge_module": "Spec", "solution_module": "Proofs", "theorem_names": [ - "Spec.clean_rules", "Spec.rules_end", "Spec.step_shortens", "Spec.rules_unique", - "Spec.normal_iff", "Spec.clean_names", + "Spec.clean_rules", "Spec.rules_end", "Spec.clean_names", "Spec.clean_shape", "Spec.clean_idem", "Spec.join_eq", "Spec.join_inside", "Spec.join_root_inside", "Spec.go_clean_tests", "Spec.go_join_tests" ],