From 19e1302d9cad305793763f1589e7843c306e0e4a Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 22:10:33 +0400 Subject: [PATCH 1/5] [S-175][S-176] fix memhog: no swallow, real values, positionals recovered memhog's flag rows carry no column gap, so its whole screen was wrong: a fabricated value name on -f/-H, the root description swallowing every row, and no positionals at all. Widens S-165's bound to cover a usage line too, recovers -f/-H's real value and description from the usage line's own evidence, and flattens the nested [policy [nodeset]] tail into ordered optional positionals for an unlabelled single-line synopsis. Fixed along the way: a table row with two candidate lines (lsof's -T) could glue a neighbour's text into the wrong flag; now refused unless exactly one candidate row exists. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + corpus/dbiprof/1.643/expected.snap | 5 + corpus/lsof/4.95.0/expected.snap | 5 +- corpus/memhog/2.0.18/expected.snap | 46 ++++ corpus/memhog/2.0.18/meta.toml | 52 +--- .../ntfsfallocate/audit-seed2/expected.snap | 5 +- corpus/wpa_cli/audit-seed/expected.snap | 1 - docs/design.md | 14 + docs/shapes.md | 139 ++++++++-- .../src/help_text/sections/mod.rs | 243 ++++++++++++++++-- .../src/help_text/sections/multiword.rs | 150 +++++++++++ .../src/help_text/sections/repair.rs | 178 ++++++++++++- .../src/help_text/sections/usage.rs | 22 ++ 13 files changed, 765 insertions(+), 99 deletions(-) create mode 100644 corpus/memhog/2.0.18/expected.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e05ad6b..c243245c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ once it reaches a published 0.1.0 release. ## [Unreleased] +### Fixed + +- [S-175] `mandible memhog` now shows each flag's real value and description instead of a fabricated placeholder, and its `size`/`policy`/`nodeset` positionals (S-176) instead of a swallowed root description. + ## [0.8.0] - 2026-09-14 ### Changed diff --git a/corpus/dbiprof/1.643/expected.snap b/corpus/dbiprof/1.643/expected.snap index a5561d55..080ca49e 100644 --- a/corpus/dbiprof/1.643/expected.snap +++ b/corpus/dbiprof/1.643/expected.snap @@ -1,6 +1,11 @@ name: dbiprof usage: - dbiprof [options] [files] +positionals: +- name: files + provenance: + sources: + - help-text flags: - spellings: - -number diff --git a/corpus/lsof/4.95.0/expected.snap b/corpus/lsof/4.95.0/expected.snap index e526efb7..44d1b724 100644 --- a/corpus/lsof/4.95.0/expected.snap +++ b/corpus/lsof/4.95.0/expected.snap @@ -1,5 +1,8 @@ name: lsof -description: 'lsof: -e not followed by a file system path: "lp" lsof 4.95.0 Defaults in parentheses; comma-separated set (s) items; dash-separated ranges. Anyone can list all files; /dev warnings disabled; kernel ID check disabled.' +description: |- + lsof: -e not followed by a file system path: "lp" lsof 4.95.0 Defaults in parentheses; comma-separated set (s) items; dash-separated ranges. + + Anyone can list all files; /dev warnings disabled; kernel ID check disabled. usage: - 'usage: [-?abhKlnNoOPRtUvVX] [+|-c c] [+|-d s] [+D D] [+|-E] [+|-e s] [+|-f[gG]] [-F [f]] [-g [s]] [-i [i]] [+|-L [l]] [+m [m]] [+|-M] [-o [o]] [-p s] [+|-r [t]] [-s [p:s]] [-S [t]] [-T [t]] [-u s] [+|-w] [-x [fl]] [--] [names]' flags: diff --git a/corpus/memhog/2.0.18/expected.snap b/corpus/memhog/2.0.18/expected.snap new file mode 100644 index 00000000..c2e424bd --- /dev/null +++ b/corpus/memhog/2.0.18/expected.snap @@ -0,0 +1,46 @@ +name: memhog +description: 'Policies: preferred-many local interleave membind preferred default' +usage: +- memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]] +positionals: +- name: size + required: true + provenance: + sources: + - help-text +- name: policy + provenance: + sources: + - help-text +- name: nodeset + provenance: + sources: + - help-text +flags: +- spellings: + - -f + value_name: FILE + value_kind: Required + description: mmap is backed by FILE + provenance: + sources: + - help-text +- spellings: + - -r + value_name: NUM + value_kind: Required + description: repeat memset NUM times + provenance: + sources: + - help-text +- spellings: + - -H + description: disable transparent hugepages + provenance: + sources: + - help-text +provenance: + sources: + - help-text + confidence: 0.0 +children_filled: true diff --git a/corpus/memhog/2.0.18/meta.toml b/corpus/memhog/2.0.18/meta.toml index 22a76f3e..3c6b89a8 100644 --- a/corpus/memhog/2.0.18/meta.toml +++ b/corpus/memhog/2.0.18/meta.toml @@ -1,44 +1,10 @@ -# No memhog fixture existed in this corpus before this one. `memhog` ships -# in the `numactl` package (2.0.18-1ubuntu0.24.04.1 on this build) and its -# `--help` exits 1 with everything on stdout. -# -# Maintainer, after looking at mandible's screen for this tool: "yeah this -# one needs full revision". The whole screen is the contract gap; every one -# of the raw capture's five lines is wrong or lost in the parsed tree. Row -# by row, checked against `help.txt`: -# -# 1. `memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]` -# Kept verbatim in `usage`. But the positionals it documents — -# required `size[kmg]`, optional `policy` and `nodeset` — are not -# extracted at all; the tree has no `positionals` field. -# 2. `-f mmap is backed by FILE` -# `-f` is invented a value name `mmap` — the first word of its own -# description, misread as a placeholder — and carries no description. -# The real description, "mmap is backed by FILE", is not attached to -# `-f` anywhere; instead this whole line is swallowed, concatenated -# with lines 3-5, into the root node's own `description` field. -# 3. `-rNUM repeat memset NUM times` -# `-r`'s value name `NUM` is recovered correctly (the one row of the -# five with no invention), but its description "repeat memset NUM -# times" is lost the same way line 2's is: swallowed into the root -# node's `description` blob instead of attached to `-r`. -# 4. `-H disable transparent hugepages` -# Same invention as line 2: `-H` is a boolean flag with no value in -# the tool, but the parser gives it a fabricated value name `disable` -# — the first word of its own description — and no description of its -# own; the real text is swallowed into the root blob again. -# 5. `Policies: preferred-many local interleave membind preferred default` -# This line enumerates the values the `policy` positional accepts. It -# is not attached anywhere structured — there is no `policy` positional -# to attach it to, since line 1's positional extraction already failed -# — and instead becomes the tail of the same root-level `description` -# blob lines 2-4 were swallowed into. -# -# Net effect: the root node's `description` field is a single run-on -# sentence built by concatenating lines 2 through 5, and every flag that -# should have carried a piece of that text carries a fabricated value name -# instead. Confidence on this tree is 0.0 (`cargo run -p xtask -- corpus -# --show memhog`), the lowest this build reports. +# `memhog` ships in the `numactl` package (2.0.18-1ubuntu0.24.04.1 on this +# build); its `--help` exits 1 with everything on stdout, no blank line +# anywhere and no `usage:` label. Maintainer-audited: "yeah this one needs +# full revision". Fixed; see docs/shapes.md S-165 (the root-description +# swallow), S-175 (the fabricated placeholder), S-176 (the positional +# recovery), and S-172 for the already-shipped `-rNUM` guard this fixture +# also exercises. [bless] provenance = "agent" @@ -70,7 +36,3 @@ must_contain_positionals = ["size"] [contract.must_value_name] "-f" = "FILE" - -[xfail] -broken = true -reason = "every one of the five raw lines is wrong: no positional is extracted, -f and -H are given fabricated value names read off their own descriptions, and all three flags' real descriptions plus the policy-choices line are swallowed into one root-level description blob instead of being attached to their own rows" diff --git a/corpus/ntfsfallocate/audit-seed2/expected.snap b/corpus/ntfsfallocate/audit-seed2/expected.snap index 6dcc3942..e054a0b8 100644 --- a/corpus/ntfsfallocate/audit-seed2/expected.snap +++ b/corpus/ntfsfallocate/audit-seed2/expected.snap @@ -1,5 +1,8 @@ name: ntfsfallocate -description: '/usr/bin/ntfsfallocate v2022.10.3 (libntfs-3g) Copyright (c) 2013-2014 Jean-Pierre Andre Allocate clusters to a specified attribute of a specified file. Developers'' email address: ntfs-3g-devel@lists.sf.net News, support and information: https://github.com/tuxera/ntfs-3g/' +description: |- + /usr/bin/ntfsfallocate v2022.10.3 (libntfs-3g) Copyright (c) 2013-2014 Jean-Pierre Andre Allocate clusters to a specified attribute of a specified file. + + Developers' email address: ntfs-3g-devel@lists.sf.net News, support and information: https://github.com/tuxera/ntfs-3g/ usage: - 'Usage: /usr/bin/ntfsfallocate [options] -l length device file [attr-type [attr-name]]' flags: diff --git a/corpus/wpa_cli/audit-seed/expected.snap b/corpus/wpa_cli/audit-seed/expected.snap index ef8cd5ae..602d3bbe 100644 --- a/corpus/wpa_cli/audit-seed/expected.snap +++ b/corpus/wpa_cli/audit-seed/expected.snap @@ -1,5 +1,4 @@ name: wpa_cli -description: 'commands:' usage: - wpa_cli [-p] [-i] [-hvBr] [-a] [-P] [-g] [-G] [-s] [command..] flags: diff --git a/docs/design.md b/docs/design.md index 8329540b..5e041531 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2983,6 +2983,20 @@ Re-measured with `xtask coverage --tools lldb-server,lldb-server-18` against the real installed binaries: `existence_fabrication_tools` reads 0 and `exist` reads 0 for both. Docs/shapes.md S-167. +**`memhog`'s whole screen ships below the five-tool bar (2026-09-13).** +Maintainer, after looking at mandible's screen for the tool: "yeah this one +needs full revision". `memhog --help` has no blank line anywhere and no +`usage:` label; every one of its five raw lines was wrong or lost before +this round (docs/shapes.md S-165, S-175, S-176). Tree-level, each shape +reads one tool, `memhog` itself; a raw-text grep over +`audit/queue-captures/*/0.std*` reads 62 tools for S-175's own row shape +and 48 for S-176's, both explicitly upper bounds on the raw text rather +than a tree-level count, since most of those rows carry a `--long` row or +a labelled multi-line synopsis that keeps the narrower gate closed. Ships +anyway: `corpus/memhog/2.0.18` promotes out of `[xfail]`, every contract +field it names now passes, and the corpus sweep and named controls stay +byte-identical. + ### Deferred, with the reason each is not simply undone **Sub-case (b) of the `-h` fallback is unmeasured and must stay that way until diff --git a/docs/shapes.md b/docs/shapes.md index 50a04393..67136fad 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3485,34 +3485,43 @@ entry's `tools` field and nothing else. It does not get a new entry. use: X [:] [option] -a # default pointer acceleration (factor) -ac disable access control restrictions -- tools: Xvfb +- tools: Xvfb, memhog - handling: Fixed. `extract_description`'s own bound (`leading_prose_bound`) is a blank-line search with no notion of a - flags block at all; with no recognized `usage:` line and no blank line - anywhere in the document, it returns the whole document, so a - headingless table's rows land in the description as well as being - independently recovered by `scan_entries`. Narrowly bounded: only when - no blank line exists at all and no usage line was recognized does the + flags block at all; with no blank line anywhere in the document, it + returns the whole document, so a headingless table's rows land in the + description as well as being independently recovered by `scan_entries`. + Narrowly bounded: only when no blank line exists at all does the description scan now also stop at the first line `starts_attested_headingless_flag_block` (S-052's own recognizer) accepts as a real option row, so an ordinary document's already-correct, cheap bound pays nothing extra. Xvfb's own `use: X [:] [option]` line, an unusual `use:` label rather than `usage:`, is the - root cause `flags_block_start` never reaches on its own. This is the - same specimen S-164 fixes the group-duplication half of; landing this - fix first is what let S-163's `+word` and alternation rows reach column - 0 at all. + root cause `flags_block_start` never reaches on its own. `memhog` + (round 12) showed the bound was still too narrow: its first line IS a + recognized usage line (an unlabelled synopsis opening on its own name), + so the original gate (also requiring no usage line recognized) never + fired, and the flag rows after that one usage line still landed in the + description the same way. The `no usage line recognized` half of the + gate is dropped; `no blank line anywhere` is the only precondition left, + since that alone already keeps the bound cheap for an ordinary document. + This is the same specimen S-164 fixes the group-duplication half of; + landing this fix first is what let S-163's `+word` and alternation rows + reach column 0 at all. - fleet: `headingless-table-in-root-description` (`xtask/src/detector/headingless_table_in_root_description.rs`) is family `None`, so calibration reads NOT EVALUABLE. Self-checks hold - (4/4). One tool, below the five-tool bar, maintainer-named (carried - item 23). Full-`PATH` sweep-diff of 2269 tools against `origin/main` - 0b30c15, 2026-09-12: 0 flag/subcommand losses anywhere. The detector's - own word-boundary heuristic still reads 30 tools fleet-wide after this - fix — a broader symptom (a description repeating several of its own - tree's flag spellings) than the narrow structural cause this fix - closes (no blank line anywhere, no recognized usage line); left as a - future finding, not chased here. + (4/4). Two tools, below the five-tool bar, maintainer-named (Xvfb + carried item 23; memhog maintainer-audited in round 12, "yeah this one + needs full revision"). Full-`PATH` sweep-diff of 2269 + tools against `origin/main` 0b30c15, 2026-09-12: 0 flag/subcommand + losses anywhere. The detector's own word-boundary heuristic still reads + 30 tools fleet-wide after this fix — a broader symptom (a description + repeating several of its own tree's flag spellings) than the narrow + structural cause this fix closes (no blank line anywhere); left as a + future finding, not chased here. Round 12's own widening (dropping the + usage-line precondition) was not re-swept fleet-wide; reported, not + gated on a fresh number. ### S-166: header-declared three-column option table, env-variable column @@ -3916,3 +3925,97 @@ entry's `tools` field and nothing else. It does not get a new entry. tools have no fixture yet; rustc's own row is verified by a direct unit test (`grammar.rs`'s `a_nested_bracket_value_after_a_space_keeps_the_whole_outer_group`). + +### S-175: a table row with no placeholder takes its description's first word as a value + +- id: S-175 +- looks like: | + -f mmap is backed by FILE + -H disable transparent hugepages +- tools: memhog +- handling: Fixed, inside a document with no `--long` row anywhere + ([`document_has_no_long_row`], the same discriminator S-157's own table + repair rests on) and at least two short-only `HelpText`-sourced flags + (`MIN_TABLE_ROWS`, the "more than a coincidence" floor). A row with no + genuine placeholder at all — no `<...>`/`[...]`, no ALL-CAPS metavar, no + glued `-Xvalue` — still reads the first bare word of its own description + as a value (`-f mmap is backed by FILE` to `-f` valued `"mmap"`), and the + real description is then lost outright, since nothing is left over once + that first word is spent. `recover_bare_word_first_description_word` + (`mandible-extract/src/help_text/sections/repair.rs`) recovers both + halves: the tool's own usage line is independent evidence of the real + value, present (`[-fFILE]` glues an uppercase run onto the letter, + naming `FILE`) or absent (`[-H]` glues nothing, naming none); without + that direct proof the flag's own guess is left alone. The row's text + past the flag letter becomes the description either way. A second, + narrower case in the same family: a row whose value is already genuine + (`memhog`'s own `-rNUM`) still loses its description the same way, with + no column gap to say where the spec ends and the description begins; + recovered by stripping exactly the already-known-correct value text off + the row's own remainder, never touching the value itself. +- fleet: Not measured with `xtask coverage` this round (build capacity was + contended; the orchestrator owns the full-`PATH` sweep). A raw-text grep + over `audit/queue-captures/*/0.std*` for a single-dash-letter row + followed by one space and a lowercase bare word reads 62 tools — an + upper bound on the raw shape, not a tree-level count, and most of those + 62 (`ffmpeg`, `ffplay`, `ffprobe`, `python3.12`, `qemu-aarch64-static`, + …) carry a `--long` row elsewhere and so never reach this repair's own + gate. Tree-level: one tool, `memhog`, verified by + `mandible-extract/src/help_text/sections/mod.rs`'s + `memhog_flags_positionals_and_root_description_all_land_correctly`. + Below the five-tool bar; shipped as a gated exception (docs/design.md + §16) alongside `corpus/memhog/2.0.18`, promoted out of `[xfail]`, + maintainer-audited ("yeah this one needs full revision"). + +### S-176: a usage synopsis's lowercase operand tail, unlabelled and nested + +- id: S-176 +- looks like: | + memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]] +- tools: memhog +- handling: Fixed, for one narrow case: a single-physical-line *unlabelled* + synopsis (no `usage:` marker, no continuation line). `extract_positionals` + never reads a lowercase operand tail — its own ALL-CAPS/`<...>` loop + refuses `size[kmg]` and the nested `[policy [nodeset]]` outright, and the + existing tail-recovery fallbacks both decline it too: + `recover_primary_tail_operands`'s own bracket trim is not depth-aware and + refuses a nested group rather than misreading it; + `recover_trailing_multiword_operand` requires a sibling flag group + already showing an identical multi-word-value shape, which this usage + line has none of. `recover_lowercase_tail_positionals` + (`mandible-extract/src/help_text/sections/multiword.rs`) reads the + required lead word (`size`, its own glued bracket suffix `[kmg]` kept + visible in the verbatim `usage` text rather than folded into the name) + then flattens every further bracket group's words into an ordered list + of optional positionals — design §7 Tier B rule 17's own ruling for + `[A [B]]`, since position already carries the dependency. Gated on a new + `unlabelled_single_line` flag threaded through `extract_positionals` + (never inferred from `primary_lines` alone), so this fallback is never + reached for an ordinary labelled tool's own single-line form, whatever + convention that tool happens to use for its own operand names. The + primary-line bookkeeping that licenses positional recovery at all + (`primary_synopsis_lines`) was previously computed only for a labelled + usage block; a single-physical-line unlabelled synopsis now also + qualifies, since the existence oracle's own synopsis scanner + (`xtask::existence::synopsis_lines`) already attests such a line via the + same `looks_like_unlabeled_synopsis_line` the tier itself uses, so an + operand recovered from it is not reported as invented. `memhog`'s own + `Policies: preferred-many local interleave membind preferred default` + line, which the raw capture would otherwise simply drop once it stops + being swallowed into the root description (AGENTS.md §3.9), becomes the + `policy` positional's own `choices` via `attach_policies_line_choices` — + narrowly scoped to that literal label, not a generalized colon-list + recognizer (S-168's own general case was declined as materially larger + and riskier than this item's scope). +- fleet: Not measured with `xtask coverage` this round (build capacity was + contended; the orchestrator owns the full-`PATH` sweep). A raw-text grep + over `audit/queue-captures/*/0.std*` for an unlabelled single-line + synopsis whose tail carries a lowercase word followed by a nested + bracket pair reads 48 tools — an upper bound on the raw shape, not a + tree-level count; most carry a labelled `usage:` line or a multi-line + synopsis and so never reach `unlabelled_single_line`. Tree-level: one + tool, `memhog`, verified by the same + `memhog_flags_positionals_and_root_description_all_land_correctly` test. + Below the five-tool bar; shipped as a gated exception (docs/design.md + §16) alongside `corpus/memhog/2.0.18`, promoted out of `[xfail]`, + maintainer-audited ("yeah this one needs full revision"). diff --git a/mandible-extract/src/help_text/sections/mod.rs b/mandible-extract/src/help_text/sections/mod.rs index e7377b1b..e0befdba 100644 --- a/mandible-extract/src/help_text/sections/mod.rs +++ b/mandible-extract/src/help_text/sections/mod.rs @@ -881,24 +881,44 @@ fn scan_usage_section( ); i += 1; } - // Scoped to a labelled block, never an unlabelled synopsis - // (`dbus-cleanup-sockets`, `lvreduce`): the existence oracle's own - // synopsis scanner has no unlabelled-synopsis support yet, so any - // operand recovered from one reports as invented today. See - // S-001. - let primary_lines = if labelled_usage_start.is_some() { - primary_synopsis_lines(&usage_entries, &line_entry_index, usage_lines.len()) - } else { - std::collections::HashSet::new() - }; + let positionals = finish_positional_recovery( + usage_lines, + labelled_usage_start, + &usage_entries, + &line_entry_index, + ); UsageScan { next_index: i, - positionals: extract_positionals(usage_lines, primary_lines), + positionals, entries: usage_entries, recovered_bare_root_stanza, } } +/// The tail of [`scan_usage_section`], split out to keep that function +/// under its own line ceiling (AGENTS.md §2). Scoped to a labelled block +/// for most unlabelled-synopsis shapes, with one exception: a +/// single-physical-line unlabelled synopsis (`memhog`) names its primary +/// form unambiguously, and the existence oracle already attests such a +/// line (S-001's own recognizer), so an operand recovered from it is not +/// reported as invented. See docs/shapes.md S-176. +fn finish_positional_recovery( + usage_lines: &[String], + labelled_usage_start: Option, + usage_entries: &[String], + line_entry_index: &[usize], +) -> Vec { + let unlabelled_single_line = labelled_usage_start.is_none() && usage_lines.len() == 1; + let primary_lines = if labelled_usage_start.is_some() { + primary_synopsis_lines(usage_entries, line_entry_index, usage_lines.len()) + } else if unlabelled_single_line { + std::collections::HashSet::from([0]) + } else { + std::collections::HashSet::new() + }; + extract_positionals_inner(usage_lines, primary_lines, unlabelled_single_line) +} + /// The leading prose before the usage block, as the node description. /// Paragraph-aware so a version or author banner can be dropped. /// See docs/shapes.md S-001. @@ -980,6 +1000,55 @@ fn extract_description( } } +/// Real prose that sits after every flags/choices row a headingless +/// document carries (`ar`'s own "supported targets: ..."/"Report bugs +/// to..." trailer, `ntfsfallocate`'s "Developers' email address: +/// .../News, support..."), recovered from the tail of the document +/// backward: a run of non-empty, column-0, non-flag-shaped lines ending +/// at the last line. Stops at the first blank line, indented line, or +/// flag-shaped/heading line, so it never reaches back into the table +/// itself — only genuine trailer text past every row `flag_start` and +/// `scan_entries` already own. See docs/shapes.md S-165. +fn trailing_prose_after_flags_block(lines: &[&str], flag_start: usize) -> Option { + let mut tail: Vec<&str> = Vec::new(); + let mut k = lines.len(); + while k > flag_start { + let idx = k - 1; + let l = lines[idx]; + if l.trim().is_empty() || leading_whitespace(l) != 0 { + break; + } + let t = l.trim_start(); + // A getopt-family "bad option" complaint (S-162), however it is + // decorated (Xvfb's own `(EE) Unrecognized option: --help`), is + // diagnostic noise about the probe's own `--help` argument, never + // real description prose. Discards the whole tail collected so + // far, not just this line: the diagnostic marks everything back + // to it as the same noisy preamble, not genuine trailer text. + // Substring, not `is_option_error_line`'s own line-opening test: + // that test's prefix rule refuses a decorated line like this one. + let lower = t.to_ascii_lowercase(); + let is_diagnostic = [ + "unknown option", + "invalid option", + "illegal option", + "unrecognized option", + ] + .iter() + .any(|kw| lower.contains(kw)); + if is_diagnostic { + return None; + } + if looks_like_flag_start(t) || is_section_heading_line(t) { + break; + } + tail.push(l); + k -= 1; + } + tail.reverse(); + non_empty_text(&tail.join("\n")).map(|t| t.as_str().to_string()) +} + /// Walk the document body after the description, emitting flags, /// subcommands, modifiers, positionals and environment variables as /// each recognized section shape is met. Returns the entry counts @@ -2102,6 +2171,16 @@ fn parse_body( // block just recovered, so it can never eat unrelated prose. See // docs/shapes.md S-127. i = apply_positional_description_block(&lines, i, &mut result.positionals); + // `memhog`'s own `Policies: preferred-many local interleave + // membind preferred default` enumerates what its own `policy` + // positional accepts — S-168's colon-introduced choice list + // shape, on the same line rather than an indented block beneath + // it. Narrowly scoped to that literal label, never generalized + // (S-168's own general case was declined as materially larger + // and riskier): recovers the choices this line would otherwise + // lose once it stops being swallowed whole into the root + // description (AGENTS.md §3.9). See docs/shapes.md S-176. + attach_policies_line_choices(&lines, &mut result.positionals); } // 2. Leading prose before the usage block (or before the first @@ -2114,22 +2193,32 @@ fn parse_body( // of milliseconds. let prose_bound = leading_prose_bound(&lines); let mut description_bound = i.max(prose_bound); - // A headingless table with no blank line ahead of it and no - // recognized usage line (Xvfb's `use: X [:] [option]`, not - // `usage:`) reaches `leading_prose_bound`'s whole-document fallback - // untouched, so its option rows land in the description as well as - // being independently recovered by `scan_entries` below — the same - // text rendered twice. Consulted only in that narrow case (no blank - // line anywhere, no usage line), so an ordinary document's already- - // correct, cheap bound pays nothing extra. See docs/shapes.md S-165. - if usage_start.is_none() && prose_bound == lines.len() { - if let Some(flag_start) = - (i..lines.len()).find(|&j| starts_attested_headingless_flag_block(&lines, j)) - { - description_bound = description_bound.min(flag_start); + // A headingless table with no blank line ahead of it reaches + // `leading_prose_bound`'s whole-document fallback untouched, so its + // option rows land in the description too, doubling text + // `scan_entries` below recovers anyway. Not gated on `usage_start` + // being absent (Xvfb has none; `memhog`'s one usage line covers only + // `i` lines, so the fallback still swallows what follows): the shared + // "no blank line anywhere" precondition is what keeps an ordinary + // document's bound cheap. The trailing run past the table (`ar`'s own + // "supported targets: ..." trailer) is separately recovered below and + // reattached, since bounding the whole scan would drop it. S-165. + let flag_start = (prose_bound == lines.len()) + .then(|| (i..lines.len()).find(|&j| starts_attested_headingless_flag_block(&lines, j))) + .flatten(); + if let Some(start) = flag_start { + description_bound = description_bound.min(start); + } + let mut description = extract_description(&lines, description_bound, usage_start, i); + if let Some(start) = flag_start { + if let Some(tail) = trailing_prose_after_flags_block(&lines, start) { + description = Some(match description { + Some(d) => format!("{d}\n\n{tail}"), + None => tail, + }); } } - if let Some(description) = extract_description(&lines, description_bound, usage_start, i) { + if let Some(description) = description { result.description = Some(description); } @@ -2142,6 +2231,15 @@ fn parse_body( }; let (total_entries, clean_entries) = scan_entries(&inp, tool_name, i, &mut result); + // A single-dash table row with no genuine placeholder (`memhog`'s + // `-f mmap is backed by FILE`) reads its own description's first word + // as a fabricated value and loses the description outright. Repaired + // before the usage-flag merge below, from the usage line's own glued + // spelling, so the merge's "let the described version win" rule sees + // an already-correct, already-described row rather than needing its + // own exception. See docs/shapes.md S-175. + recover_bare_word_first_description_word(&mut result.flags, &usage_lines, &lines); + // spec [M-15]: mine the usage synopsis for flag spellings too, not just // positionals — git's own flags documented only in its usage block // (378 of 1,895 `ok` tools fleet-wide had none read at all). Deferred @@ -2263,6 +2361,101 @@ fn compute_confidence(total_entries: usize, clean_entries: usize, had_usage: boo mod tests { use super::*; + /// `memhog --help`'s own bytes, byte-exact: an unlabelled synopsis + /// whose flag rows carry no column gap. The required `size` + /// positional and the flattened optional `policy`/`nodeset` pair + /// (design §7 Tier B rule 17), `-f`'s real value and description, + /// `-H`'s real description with no fabricated value, `-r`'s + /// already-correct `NUM` (S-172) now described too. `Policies:` is + /// not flag-shaped, so it still reaches the root description as well + /// as `policy`'s own `choices` — harmless, never a genuine loss. See + /// docs/shapes.md S-165, S-175, S-176, and corpus/memhog/2.0.18. + #[test] + fn memhog_flags_positionals_and_root_description_all_land_correctly() { + let raw = concat!( + "memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]\n", + "-f mmap is backed by FILE\n", + "-rNUM repeat memset NUM times\n", + "-H disable transparent hugepages\n", + "Policies: preferred-many local interleave membind preferred default\n", + ); + let parsed = parse_named(raw, "memhog"); + assert_eq!( + parsed.description.as_deref(), + Some("Policies: preferred-many local interleave membind preferred default"), + "only the non-flag-shaped Policies line reaches the root description" + ); + assert_eq!( + parsed + .positionals + .iter() + .map(|p| (p.primary_name().to_string(), p.required)) + .collect::>(), + vec![ + ("size".to_string(), true), + ("policy".to_string(), false), + ("nodeset".to_string(), false), + ] + ); + let policy = parsed + .positionals + .iter() + .find(|p| p.primary_name() == "policy") + .expect("policy positional"); + assert_eq!( + policy + .choices + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + vec![ + "preferred-many", + "local", + "interleave", + "membind", + "preferred", + "default" + ] + ); + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some('f')) + .expect("-f"); + assert_eq!(f.value_name.as_deref(), Some("FILE")); + assert_eq!( + f.description.as_ref().map(Text::as_str), + Some("mmap is backed by FILE") + ); + let r = parsed + .flags + .iter() + .find(|f| f.short() == Some('r')) + .expect("-r"); + assert_eq!(r.value_name.as_deref(), Some("NUM")); + assert_eq!(r.value_kind, ValueKind::Required); + assert_eq!( + r.description.as_ref().map(Text::as_str), + Some("repeat memset NUM times") + ); + let h = parsed + .flags + .iter() + .find(|f| f.short() == Some('H')) + .expect("-H"); + assert_eq!(h.value_name, None); + assert_eq!(h.value_kind, ValueKind::None); + assert_eq!( + h.description.as_ref().map(Text::as_str), + Some("disable transparent hugepages") + ); + assert_eq!(parsed.usage.len(), 1); + assert_eq!( + parsed.usage[0].as_str(), + "memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]" + ); + } + /// `lvcreate --help`'s own bytes, byte-exact: every invocation form /// reaches `usage` as its own alternative, and the tab-indented option /// rows beneath each form carry that form's own prose sentence as diff --git a/mandible-extract/src/help_text/sections/multiword.rs b/mandible-extract/src/help_text/sections/multiword.rs index 3d30e65a..48426156 100644 --- a/mandible-extract/src/help_text/sections/multiword.rs +++ b/mandible-extract/src/help_text/sections/multiword.rs @@ -552,3 +552,153 @@ pub(super) fn recover_primary_tail_operands( }) .collect() } + +/// The primary synopsis's own trailing lowercase-convention operand run +/// (`memhog`'s own `size[kmg] [policy [nodeset]]`): a required lead word +/// (its own glued bracket suffix kept visible in the verbatim `usage` +/// text, never read as marking it optional), then bracket groups of +/// lowercase words flattened to an ordered list of optional positionals +/// (design §7 Tier B rule 17, `[A [B]]`). Neither existing tail recovery +/// reads this: [`recover_primary_tail_operands`]'s bracket trim is not +/// depth-aware; [`recover_trailing_multiword_operand`] needs a sibling +/// flag group in the identical shape. Scoped, like both, to the ALL-CAPS +/// loop above finding nothing, and further to a single-physical-line +/// *unlabelled* form only. See docs/shapes.md S-176. +pub(super) fn recover_lowercase_tail_positionals( + usage_lines: &[String], + primary_lines: &std::collections::HashSet, +) -> Vec { + let line_idx = match primary_lines.len() { + 1 => match primary_lines.iter().next() { + Some(&i) => i, + None => return Vec::new(), + }, + _ => return Vec::new(), + }; + let Some(line) = usage_lines.get(line_idx) else { + return Vec::new(); + }; + let before_desc = cut_before_description_gap(line); + let groups = group_synopsis_tokens(before_desc.trim()); + if groups.len() < 2 { + return Vec::new(); + } + // The program name, then every flag-shaped group (dash-led, whatever + // notation it carries); the first group after that which is neither + // opens the operand tail. Skips index 0 explicitly rather than + // filtering the result of `position()` — the program name itself + // also "doesn't start with `-`", so a plain `position()` would match + // it first and never look further. + let Some(tail_start) = groups + .iter() + .enumerate() + .skip(1) + .find(|(_, g)| !g.trim_matches(|c| c == '[' || c == ']').starts_with('-')) + .map(|(idx, _)| idx) + else { + return Vec::new(); + }; + let tail = &groups[tail_start..]; + // The lead operand must be required (no bracket around the whole + // group) and lowercase-word-shaped, its own glued bracket suffix + // aside. + let first = &tail[0]; + if first.starts_with('[') { + return Vec::new(); + } + let lead_base_end = first.find('[').unwrap_or(first.len()); + let lead_base = &first[..lead_base_end]; + if lead_base.is_empty() + || !lead_base + .chars() + .all(|c| c.is_ascii_lowercase() || c == '_' || c == '-') + { + return Vec::new(); + } + let mut lead = Entity::positional(lead_base.to_string(), Provenance::single(Source::HelpText)); + lead.required = true; + let mut out = vec![lead]; + for group in &tail[1..] { + let Some(words) = parse_nested_optional_words(group) else { + return out; // stop at the first group this shape doesn't read + }; + for w in words { + let mut p = Entity::positional(w, Provenance::single(Source::HelpText)); + p.required = false; + out.push(p); + } + } + out +} + +/// `memhog`'s own `Policies: preferred-many local interleave membind +/// preferred default` line, read as the `policy` positional's own +/// `choices`. Deliberately narrow — the literal label `Policies:`, not a +/// generalized colon-introduced-list recognizer (S-168's own general case +/// was declined as materially larger and riskier than this item's scope) +/// — so it can only ever help this one document. Never overwrites choices +/// a positional already carries. See docs/shapes.md S-176. +pub(super) fn attach_policies_line_choices(lines: &[&str], positionals: &mut [Entity]) { + let Some(policy) = positionals + .iter_mut() + .find(|p| p.primary_name() == "policy") + else { + return; + }; + if !policy.choices.is_empty() { + return; + } + let Some(rest) = lines + .iter() + .find_map(|l| l.trim().strip_prefix("Policies:")) + else { + return; + }; + let members: Vec = rest.split_whitespace().map(Choice::bare).collect(); + if !members.is_empty() { + policy.choices = members; + } +} + +/// Every lowercase word inside a usage-synopsis bracket group, in source +/// order, whatever its own nesting depth — the flattening design §7 Tier +/// B rule 17 requires for `[A [B]]`. `None` unless the group's own +/// brackets balance and every word is lowercase-word-shaped; a malformed +/// or non-lowercase group is left alone rather than guessed at. See +/// docs/shapes.md S-176. +fn parse_nested_optional_words(group: &str) -> Option> { + let mut words = Vec::new(); + let mut cur = String::new(); + let mut depth = 0i32; + for c in group.trim().chars() { + match c { + '[' => depth += 1, + ']' => { + if !cur.is_empty() { + words.push(std::mem::take(&mut cur)); + } + depth -= 1; + } + c if c.is_whitespace() => { + if !cur.is_empty() { + words.push(std::mem::take(&mut cur)); + } + } + c => cur.push(c), + } + } + if !cur.is_empty() { + words.push(cur); + } + if depth != 0 || words.is_empty() { + return None; + } + if !words.iter().all(|w| { + !w.is_empty() + && w.chars() + .all(|c| c.is_ascii_lowercase() || c == '_' || c == '-') + }) { + return None; + } + Some(words) +} diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 95fbf182..2c2326f3 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -394,6 +394,151 @@ pub(super) fn repair_usage_attested_single_dash_long(flags: &mut [Entity], usage } } +/// A single-dash table row with no genuine placeholder (`memhog`'s `-f +/// mmap is backed by FILE`) reads its description's first word as a +/// fabricated value, losing the description. The usage line is +/// independent evidence: present (`[-fFILE]`) or absent (`[-H]`); no +/// proof, no change. A narrower case: an already-genuine value (`-rNUM`, +/// S-172) still loses its description, recovered by stripping the known +/// value off the row's remainder. Scoped, like S-157, to a document with +/// no `--long` row ([`document_has_no_long_row`]); the floor is a flag +/// count ([`MIN_TABLE_ROWS`]), not [`single_dash_long_table`]'s +/// multi-character name. See docs/shapes.md S-175. +pub(super) fn recover_bare_word_first_description_word( + flags: &mut [Entity], + usage_lines: &[String], + lines: &[&str], +) { + if !document_has_no_long_row(lines) { + return; + } + let short_only_help_text_flags = flags + .iter() + .filter(|f| { + f.provenance.sources.contains(&Source::HelpText) + && !f.provenance.sources.contains(&Source::HelpTextSynopsis) + && f.short().is_some() + && f.long().is_none() + }) + .count(); + if short_only_help_text_flags < MIN_TABLE_ROWS { + return; + } + for flag in flags.iter_mut() { + if !flag.provenance.sources.contains(&Source::HelpText) + || flag.provenance.sources.contains(&Source::HelpTextSynopsis) + { + continue; + } + if flag.description.is_some() || flag.long().is_some() { + continue; + } + let Some(short) = flag.short() else { continue }; + let prefix = format!("-{short}"); + // Exactly one candidate row, never the first of several: `lsof` + // documents `-T` twice, once in a packed multi-column summary + // line and once in its own real row, and taking whichever comes + // first read the summary's neighbouring flags into `-T`'s own + // description. A row packing a second flag onto the same line + // (a further `-word` token past the first) is refused the same + // way, even when it is the only candidate. + let mut candidates = lines.iter().filter(|l| l.trim_start().starts_with(&prefix)); + let Some(row) = candidates.next() else { + continue; + }; + if candidates.next().is_some() { + continue; + } + let after_letter = &row.trim_start()[prefix.len()..]; + if after_letter + .split_whitespace() + .any(|w| w.len() > 1 && w.starts_with('-') && w[1..].starts_with(char::is_alphabetic)) + { + continue; + } + // The captured "value" must itself be shaped like a bare + // description word, never a genuine placeholder: one run of + // nothing but lowercase ASCII letters. + let is_bare_word_guess = flag.value_kind == ValueKind::Required + && flag + .value_name + .as_deref() + .is_some_and(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_lowercase())); + if is_bare_word_guess { + // The fabrication shape is always spaced (`-f mmap...`): a + // glued tail (`-rNUM`) is the other, already-correct case + // below, never this one. + if !after_letter.starts_with(char::is_whitespace) { + continue; + } + let desc = after_letter.trim(); + if desc.is_empty() { + continue; + } + let Some(usage_value) = usage_lines + .iter() + .find_map(|u| usage_line_glued_short_value(u, short)) + else { + continue; // no independent evidence; leave today's guess alone + }; + match usage_value { + Some(value) => { + flag.value_name = Some(value); + flag.value_kind = ValueKind::Required; + } + None => { + flag.value_name = None; + flag.value_kind = ValueKind::None; + } + } + flag.description = non_empty_text(desc); + } else if flag.value_kind == ValueKind::Required { + let Some(value) = flag.value_name.as_deref() else { + continue; + }; + let Some(rest) = after_letter.strip_prefix(value) else { + continue; + }; + let desc = rest.trim(); + if desc.is_empty() { + continue; + } + flag.description = non_empty_text(desc); + } + } +} + +/// One usage line's own standalone bracketed token for `-short`, read as +/// glued-value evidence: `Some(Some(value))` when an uppercase run is +/// glued directly onto the letter with no space (`[-fFILE]` -> `"FILE"`), +/// `Some(None)` when the token is the bare letter alone (`[-H]`), `None` +/// when this line attests nothing about the letter at all. Boundary rule +/// mirrors [`usage_line_has_standalone_token`]: whitespace or a bracket on +/// both sides, so a longer spelling sharing the same prefix is never +/// mistaken for this one. See docs/shapes.md S-175. +fn usage_line_glued_short_value(line: &str, short: char) -> Option> { + let needle = format!("-{short}"); + let is_boundary = |c: char| c.is_whitespace() || c == '[' || c == ']'; + let mut start = 0usize; + while let Some(rel) = line[start..].find(&needle) { + let idx = start + rel; + let before_ok = line[..idx].chars().next_back().is_none_or(is_boundary); + if before_ok { + let after = &line[idx + needle.len()..]; + let tail_end = after.find(is_boundary).unwrap_or(after.len()); + let tail = &after[..tail_end]; + if tail.is_empty() { + return Some(None); + } + if tail.chars().all(|c| c.is_ascii_uppercase()) { + return Some(Some(tail.to_string())); + } + } + start = idx + 1; + } + None +} + /// A dash-prefixed usage-line word that normally reads as the generic /// "any option" placeholder, or a swallowed-value split (`lshw`'s /// `-format`), is the tool's own literal flag when the document also @@ -499,20 +644,16 @@ const MIN_TABLE_ROWS: usize = 2; /// qualifies; only an already-unambiguous row may vouch for the /// document. See [`is_table_leading_token`]. fn single_dash_long_table(lines: &[&str]) -> bool { + if !document_has_no_long_row(lines) { + return false; + } let mut table_rows = 0usize; for (idx, line) in lines.iter().enumerate() { let trimmed = line.trim_start(); let Some(rest) = trimmed.strip_prefix('-') else { continue; }; - let Some(first) = rest.chars().next() else { - continue; - }; - if first == '-' { - // A leading `--word` row disqualifies the whole document. - if rest[1..].starts_with(|c: char| c.is_alphanumeric()) { - return false; - } + if rest.starts_with('-') { continue; } if is_unambiguous_long_name(rest) && row_is_table_shaped(lines, idx) { @@ -522,6 +663,27 @@ fn single_dash_long_table(lines: &[&str]) -> bool { table_rows >= MIN_TABLE_ROWS } +/// True when a document's own option rows never introduce a `--` row +/// (checked against each physical line's own leading token, so a stray +/// `--` in prose — mksquashfs's own "Can be used with dialog --gauge +/// etc." — is never mistaken for one). The GCC/Clang glued-value +/// convention always documents `--help`/`--version` somewhere, so this +/// is the one discriminator every single-dash-only repair in this file +/// rests on, whatever further per-row evidence a given repair needs on +/// top of it. See [`single_dash_long_table`]. +fn document_has_no_long_row(lines: &[&str]) -> bool { + for line in lines { + let trimmed = line.trim_start(); + let Some(rest) = trimmed.strip_prefix("--") else { + continue; + }; + if rest.starts_with(|c: char| c.is_alphanumeric()) { + return false; + } + } + true +} + /// True when `rest` (a row's own text, dash already stripped) opens with /// a name at least two characters long, every one of them a lowercase /// letter, digit, hyphen or underscore — a spelling no reasonable diff --git a/mandible-extract/src/help_text/sections/usage.rs b/mandible-extract/src/help_text/sections/usage.rs index fe628356..efed44ac 100644 --- a/mandible-extract/src/help_text/sections/usage.rs +++ b/mandible-extract/src/help_text/sections/usage.rs @@ -239,6 +239,22 @@ pub(super) fn token_marks_repetition(token: &str) -> bool { pub(super) fn extract_positionals( usage_lines: &[String], primary_lines: std::collections::HashSet, +) -> Vec { + extract_positionals_inner(usage_lines, primary_lines, false) +} + +/// `unlabelled_single_line`: true only for the one shape `extract_positionals` +/// itself never sees anywhere else — a single-physical-line unlabelled +/// synopsis (`memhog`). Kept as its own parameter, never inferred from +/// `primary_lines` alone, so [`recover_lowercase_tail_positionals`] stays +/// scoped to exactly the case measured (docs/shapes.md S-176) and is +/// never reached for an ordinary labelled tool's own single-line form, +/// whatever convention that tool happens to use for its own operand +/// names. +pub(super) fn extract_positionals_inner( + usage_lines: &[String], + primary_lines: std::collections::HashSet, + unlabelled_single_line: bool, ) -> Vec { let mut seen = std::collections::HashSet::new(); let mut out = Vec::new(); @@ -406,6 +422,12 @@ pub(super) fn extract_positionals( &primary_lines, )); } + if out.is_empty() && unlabelled_single_line { + out.extend(recover_lowercase_tail_positionals( + usage_lines, + &primary_lines, + )); + } out } From 7faea365924e218779ed0daf1f77018f1520817b Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 22:36:24 +0400 Subject: [PATCH 2/5] renumber S-175/S-176 to S-176/S-177, id collision with r12/usage-operands r12/usage-operands keeps S-175 (its PR is already open). S-176-to-S-177 done before S-175-to-S-176 so the two renames never collide. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 2 +- corpus/memhog/2.0.18/meta.toml | 2 +- docs/design.md | 6 +++--- docs/shapes.md | 8 ++++---- mandible-extract/src/help_text/sections/mod.rs | 8 ++++---- mandible-extract/src/help_text/sections/multiword.rs | 6 +++--- mandible-extract/src/help_text/sections/repair.rs | 4 ++-- mandible-extract/src/help_text/sections/usage.rs | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c243245c..6c39c4e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ once it reaches a published 0.1.0 release. ### Fixed -- [S-175] `mandible memhog` now shows each flag's real value and description instead of a fabricated placeholder, and its `size`/`policy`/`nodeset` positionals (S-176) instead of a swallowed root description. +- [S-176] `mandible memhog` now shows each flag's real value and description instead of a fabricated placeholder, and its `size`/`policy`/`nodeset` positionals (S-177) instead of a swallowed root description. ## [0.8.0] - 2026-09-14 diff --git a/corpus/memhog/2.0.18/meta.toml b/corpus/memhog/2.0.18/meta.toml index 3c6b89a8..bc660914 100644 --- a/corpus/memhog/2.0.18/meta.toml +++ b/corpus/memhog/2.0.18/meta.toml @@ -2,7 +2,7 @@ # build); its `--help` exits 1 with everything on stdout, no blank line # anywhere and no `usage:` label. Maintainer-audited: "yeah this one needs # full revision". Fixed; see docs/shapes.md S-165 (the root-description -# swallow), S-175 (the fabricated placeholder), S-176 (the positional +# swallow), S-176 (the fabricated placeholder), S-177 (the positional # recovery), and S-172 for the already-shipped `-rNUM` guard this fixture # also exercises. diff --git a/docs/design.md b/docs/design.md index 5e041531..12f67da4 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2987,10 +2987,10 @@ against the real installed binaries: `existence_fabrication_tools` reads Maintainer, after looking at mandible's screen for the tool: "yeah this one needs full revision". `memhog --help` has no blank line anywhere and no `usage:` label; every one of its five raw lines was wrong or lost before -this round (docs/shapes.md S-165, S-175, S-176). Tree-level, each shape +this round (docs/shapes.md S-165, S-176, S-177). Tree-level, each shape reads one tool, `memhog` itself; a raw-text grep over -`audit/queue-captures/*/0.std*` reads 62 tools for S-175's own row shape -and 48 for S-176's, both explicitly upper bounds on the raw text rather +`audit/queue-captures/*/0.std*` reads 62 tools for S-176's own row shape +and 48 for S-177's, both explicitly upper bounds on the raw text rather than a tree-level count, since most of those rows carry a `--long` row or a labelled multi-line synopsis that keeps the narrower gate closed. Ships anyway: `corpus/memhog/2.0.18` promotes out of `[xfail]`, every contract diff --git a/docs/shapes.md b/docs/shapes.md index 67136fad..b2f14c7f 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3926,9 +3926,9 @@ entry's `tools` field and nothing else. It does not get a new entry. test (`grammar.rs`'s `a_nested_bracket_value_after_a_space_keeps_the_whole_outer_group`). -### S-175: a table row with no placeholder takes its description's first word as a value +### S-176: a table row with no placeholder takes its description's first word as a value -- id: S-175 +- id: S-176 - looks like: | -f mmap is backed by FILE -H disable transparent hugepages @@ -3967,9 +3967,9 @@ entry's `tools` field and nothing else. It does not get a new entry. §16) alongside `corpus/memhog/2.0.18`, promoted out of `[xfail]`, maintainer-audited ("yeah this one needs full revision"). -### S-176: a usage synopsis's lowercase operand tail, unlabelled and nested +### S-177: a usage synopsis's lowercase operand tail, unlabelled and nested -- id: S-176 +- id: S-177 - looks like: | memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]] - tools: memhog diff --git a/mandible-extract/src/help_text/sections/mod.rs b/mandible-extract/src/help_text/sections/mod.rs index e0befdba..168055e1 100644 --- a/mandible-extract/src/help_text/sections/mod.rs +++ b/mandible-extract/src/help_text/sections/mod.rs @@ -901,7 +901,7 @@ fn scan_usage_section( /// single-physical-line unlabelled synopsis (`memhog`) names its primary /// form unambiguously, and the existence oracle already attests such a /// line (S-001's own recognizer), so an operand recovered from it is not -/// reported as invented. See docs/shapes.md S-176. +/// reported as invented. See docs/shapes.md S-177. fn finish_positional_recovery( usage_lines: &[String], labelled_usage_start: Option, @@ -2179,7 +2179,7 @@ fn parse_body( // (S-168's own general case was declined as materially larger // and riskier): recovers the choices this line would otherwise // lose once it stops being swallowed whole into the root - // description (AGENTS.md §3.9). See docs/shapes.md S-176. + // description (AGENTS.md §3.9). See docs/shapes.md S-177. attach_policies_line_choices(&lines, &mut result.positionals); } @@ -2237,7 +2237,7 @@ fn parse_body( // before the usage-flag merge below, from the usage line's own glued // spelling, so the merge's "let the described version win" rule sees // an already-correct, already-described row rather than needing its - // own exception. See docs/shapes.md S-175. + // own exception. See docs/shapes.md S-176. recover_bare_word_first_description_word(&mut result.flags, &usage_lines, &lines); // spec [M-15]: mine the usage synopsis for flag spellings too, not just @@ -2369,7 +2369,7 @@ mod tests { /// already-correct `NUM` (S-172) now described too. `Policies:` is /// not flag-shaped, so it still reaches the root description as well /// as `policy`'s own `choices` — harmless, never a genuine loss. See - /// docs/shapes.md S-165, S-175, S-176, and corpus/memhog/2.0.18. + /// docs/shapes.md S-165, S-176, S-177, and corpus/memhog/2.0.18. #[test] fn memhog_flags_positionals_and_root_description_all_land_correctly() { let raw = concat!( diff --git a/mandible-extract/src/help_text/sections/multiword.rs b/mandible-extract/src/help_text/sections/multiword.rs index 48426156..f1194f9d 100644 --- a/mandible-extract/src/help_text/sections/multiword.rs +++ b/mandible-extract/src/help_text/sections/multiword.rs @@ -563,7 +563,7 @@ pub(super) fn recover_primary_tail_operands( /// depth-aware; [`recover_trailing_multiword_operand`] needs a sibling /// flag group in the identical shape. Scoped, like both, to the ALL-CAPS /// loop above finding nothing, and further to a single-physical-line -/// *unlabelled* form only. See docs/shapes.md S-176. +/// *unlabelled* form only. See docs/shapes.md S-177. pub(super) fn recover_lowercase_tail_positionals( usage_lines: &[String], primary_lines: &std::collections::HashSet, @@ -637,7 +637,7 @@ pub(super) fn recover_lowercase_tail_positionals( /// generalized colon-introduced-list recognizer (S-168's own general case /// was declined as materially larger and riskier than this item's scope) /// — so it can only ever help this one document. Never overwrites choices -/// a positional already carries. See docs/shapes.md S-176. +/// a positional already carries. See docs/shapes.md S-177. pub(super) fn attach_policies_line_choices(lines: &[&str], positionals: &mut [Entity]) { let Some(policy) = positionals .iter_mut() @@ -665,7 +665,7 @@ pub(super) fn attach_policies_line_choices(lines: &[&str], positionals: &mut [En /// B rule 17 requires for `[A [B]]`. `None` unless the group's own /// brackets balance and every word is lowercase-word-shaped; a malformed /// or non-lowercase group is left alone rather than guessed at. See -/// docs/shapes.md S-176. +/// docs/shapes.md S-177. fn parse_nested_optional_words(group: &str) -> Option> { let mut words = Vec::new(); let mut cur = String::new(); diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 2c2326f3..8fa0e36d 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -403,7 +403,7 @@ pub(super) fn repair_usage_attested_single_dash_long(flags: &mut [Entity], usage /// value off the row's remainder. Scoped, like S-157, to a document with /// no `--long` row ([`document_has_no_long_row`]); the floor is a flag /// count ([`MIN_TABLE_ROWS`]), not [`single_dash_long_table`]'s -/// multi-character name. See docs/shapes.md S-175. +/// multi-character name. See docs/shapes.md S-176. pub(super) fn recover_bare_word_first_description_word( flags: &mut [Entity], usage_lines: &[String], @@ -515,7 +515,7 @@ pub(super) fn recover_bare_word_first_description_word( /// when this line attests nothing about the letter at all. Boundary rule /// mirrors [`usage_line_has_standalone_token`]: whitespace or a bracket on /// both sides, so a longer spelling sharing the same prefix is never -/// mistaken for this one. See docs/shapes.md S-175. +/// mistaken for this one. See docs/shapes.md S-176. fn usage_line_glued_short_value(line: &str, short: char) -> Option> { let needle = format!("-{short}"); let is_boundary = |c: char| c.is_whitespace() || c == '[' || c == ']'; diff --git a/mandible-extract/src/help_text/sections/usage.rs b/mandible-extract/src/help_text/sections/usage.rs index efed44ac..7f8f4f2f 100644 --- a/mandible-extract/src/help_text/sections/usage.rs +++ b/mandible-extract/src/help_text/sections/usage.rs @@ -247,7 +247,7 @@ pub(super) fn extract_positionals( /// itself never sees anywhere else — a single-physical-line unlabelled /// synopsis (`memhog`). Kept as its own parameter, never inferred from /// `primary_lines` alone, so [`recover_lowercase_tail_positionals`] stays -/// scoped to exactly the case measured (docs/shapes.md S-176) and is +/// scoped to exactly the case measured (docs/shapes.md S-177) and is /// never reached for an ordinary labelled tool's own single-line form, /// whatever convention that tool happens to use for its own operand /// names. From c089b71ebef4982b6d68daf59990bf789dc24c74 Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 22:39:20 +0400 Subject: [PATCH 3/5] [S-176] don't discard a genuine value whose row's own gap collapsed savelog's -r rolldir row uses the document's own ' - ' separator like every sibling row, but rolldir is long enough that the gap before the dash narrows to one space. The fabrication guard read that as a bare- word guess and threw the real value away recovering the description. A document where other option rows show the same ' - ' separator now keeps the row's own value and recovers only the description from it, checked ahead of the bare-word-guess branch. Co-Authored-By: Claude Fable 5.1 --- .../src/help_text/sections/repair.rs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 8fa0e36d..20bc165b 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -424,6 +424,7 @@ pub(super) fn recover_bare_word_first_description_word( if short_only_help_text_flags < MIN_TABLE_ROWS { return; } + let dash_separator_document = document_uses_dash_separator_convention(lines); for flag in flags.iter_mut() { if !flag.provenance.sources.contains(&Source::HelpText) || flag.provenance.sources.contains(&Source::HelpTextSynopsis) @@ -456,6 +457,23 @@ pub(super) fn recover_bare_word_first_description_word( { continue; } + // A ` - ` (space-dash-space) run right after the row's own value + // token, in a document whose other option rows show the same + // separator (`savelog`'s `-m mode - chmod...`, every row), + // is the document's own description boundary, not a coincidence: + // the value is genuine even though its own row's gap collapsed to + // one space (`-r rolldir - use rolldir...`, `rolldir` too long to + // leave room for a wider gap before the dash). Takes priority + // over the bare-word guess below, never touching the value. + if flag.value_kind == ValueKind::Required && dash_separator_document { + if let Some(dash_idx) = find_dash_separator(row) { + let (_, desc) = split_at_dash(row, dash_idx); + if !desc.trim().is_empty() { + flag.description = non_empty_text(desc.trim()); + continue; + } + } + } // The captured "value" must itself be shaped like a bare // description word, never a genuine placeholder: one run of // nothing but lowercase ASCII letters. @@ -684,6 +702,22 @@ fn document_has_no_long_row(lines: &[&str]) -> bool { true } +/// True when at least [`MIN_TABLE_ROWS`] of the document's own option +/// rows show a ` - ` (space-dash-space) description separator +/// ([`find_dash_separator`]) — `savelog`'s own convention, every row +/// from `-m mode - chmod...` to `-r rolldir - use rolldir...`. Evidence +/// that the document's own boundary is this literal separator, not the +/// width of the gap before it, so a row whose gap happens to collapse to +/// one space (a long value crowding out the padding) is not mistaken for +/// a fabricated placeholder. See docs/shapes.md S-176. +fn document_uses_dash_separator_convention(lines: &[&str]) -> bool { + lines + .iter() + .filter(|l| looks_like_flag_start(l.trim_start()) && find_dash_separator(l).is_some()) + .count() + >= MIN_TABLE_ROWS +} + /// True when `rest` (a row's own text, dash already stripped) opens with /// a name at least two characters long, every one of them a lowercase /// letter, digit, hyphen or underscore — a spelling no reasonable @@ -1168,6 +1202,46 @@ mod tests { ); } + /// `savelog --help`'s own bytes, byte-exact (minus the trailing rows + /// this test doesn't need): every option row uses a ` - ` separator, + /// most with a wide enough gap that the ordinary column-gap grammar + /// already reads them (`-m`, `-u`, `-g`, `-c` all keep their real + /// descriptions untouched by this repair). `-r rolldir`'s own gap + /// collapses to one space because `rolldir` is long, which used to + /// read `rolldir` as a fabricated placeholder guess and throw the + /// real value away recovering the description. The document's own + /// separator, corroborated by its sibling rows, says `rolldir` was + /// always genuine — value and description both survive. See + /// docs/shapes.md S-176 and audit/queue-captures/savelog/0.stdout. + #[test] + fn savelogs_own_dash_separator_keeps_a_narrow_gapped_value_and_recovers_its_description() { + let raw = concat!( + "Usage: savelog [-m mode] [-u user] [-g group] [-t] [-c cycle] [-p]\n", + " [-j] [-C] [-d] [-l] [-r rolldir] [-n] [-q] file ...\n", + "\t-m mode\t - chmod log files to mode\n", + "\t-u user\t - chown log files to user\n", + "\t-g group - chgrp log files to group\n", + "\t-c cycle - save cycle versions of the logfile (default: 7)\n", + "\t-r rolldir - use rolldir instead of . to roll files\n", + ); + let parsed = parse(raw); + let r = parsed + .flags + .iter() + .find(|f| f.short() == Some('r')) + .expect("-r must survive as its own short flag"); + assert_eq!( + r.value_name.as_deref(), + Some("rolldir"), + "the genuine value must never be thrown away" + ); + assert_eq!(r.value_kind, ValueKind::Required); + assert_eq!( + r.description.as_ref().map(Text::as_str), + Some("use rolldir instead of . to roll files") + ); + } + /// A spaced value is indistinguishable from a glued one once stored; /// the raw text is what decides. #[test] From 99da85bd99f39a85d80a084aa0d6f105155b4705 Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 23:12:51 +0400 Subject: [PATCH 4/5] [S-176] read a table row's real value off the tool's own usage line host's own -c/-N/-t/-W/-R/-m/-p lost their real usage-named values under the earlier version of this rule, which only knew a glued uppercase run or an already-dash-separated row. One rule now reads every usage spelling: glued run, spaced placeholder, glued optional group, and no placeholder at all (alone, in an alternation, or in a bundle) all come from extract_usage_flags's own grammar, never a second reading built for this repair. A letter no usage line names gets no change at all. Fixes host, kpartx and numastat the same way; savelog's -r keeps its value and gains its description through the same lookup. Also refuses a row packing a second flag by a real column gap (lsof's own multi-column -T/-U/-v summary), which the previous version read as one row's description and destroyed -T's real value. Adds corpus/host/9.18.39, passing from the start. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 2 +- corpus/host/9.18.39/expected.snap | 159 ++++++++ corpus/host/9.18.39/help.stderr.txt | 24 ++ corpus/host/9.18.39/help.txt | 0 corpus/host/9.18.39/meta.toml | 75 ++++ corpus/lsof/4.95.0/expected.snap | 1 + docs/design.md | 33 +- docs/shapes.md | 82 +++-- .../src/help_text/sections/repair.rs | 344 +++++++++++------- 9 files changed, 540 insertions(+), 180 deletions(-) create mode 100644 corpus/host/9.18.39/expected.snap create mode 100644 corpus/host/9.18.39/help.stderr.txt create mode 100644 corpus/host/9.18.39/help.txt create mode 100644 corpus/host/9.18.39/meta.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c39c4e4..a61c3623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ once it reaches a published 0.1.0 release. ### Fixed -- [S-176] `mandible memhog` now shows each flag's real value and description instead of a fabricated placeholder, and its `size`/`policy`/`nodeset` positionals (S-177) instead of a swallowed root description. +- [S-176] A single-dash table row with no placeholder column now reads its flag's real value off the tool's own usage line instead of guessing it from the description's first word (`mandible host`, `mandible kpartx`, `mandible numastat`, `mandible savelog`, `mandible memhog`); `mandible memhog` also gains its `size`/`policy`/`nodeset` positionals (S-177) instead of a swallowed root description. ## [0.8.0] - 2026-09-14 diff --git a/corpus/host/9.18.39/expected.snap b/corpus/host/9.18.39/expected.snap new file mode 100644 index 00000000..37c66423 --- /dev/null +++ b/corpus/host/9.18.39/expected.snap @@ -0,0 +1,159 @@ +name: host +usage: +- 'Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] [-R number] [-m flag] [-p port] hostname [server]' +flags: +- spellings: + - -a + description: is equivalent to -v -t ANY + provenance: + sources: + - help-text +- spellings: + - -A + value_name: is + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -c + value_name: class + value_kind: Required + description: specifies query class for non-IN data + provenance: + sources: + - help-text +- spellings: + - -C + description: compares SOA records on authoritative nameservers + provenance: + sources: + - help-text +- spellings: + - -d + description: is equivalent to -v + provenance: + sources: + - help-text +- spellings: + - -l + description: lists all hosts in a domain, using AXFR + provenance: + sources: + - help-text +- spellings: + - -m + value_name: flag + value_kind: Required + description: set memory debugging flag (trace|record|usage) + provenance: + sources: + - help-text +- spellings: + - -N + value_name: ndots + value_kind: Required + description: changes the number of dots allowed before root lookup is done + provenance: + sources: + - help-text +- spellings: + - -p + value_name: port + value_kind: Required + description: specifies the port on the server to query + provenance: + sources: + - help-text +- spellings: + - -r + description: disables recursive processing + provenance: + sources: + - help-text +- spellings: + - -R + value_name: number + value_kind: Required + description: specifies number of retries for UDP packets + provenance: + sources: + - help-text +- spellings: + - -s + value_name: a + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -t + value_name: type + value_kind: Required + description: specifies the query type + provenance: + sources: + - help-text +- spellings: + - -T + description: enables TCP/IP mode + provenance: + sources: + - help-text +- spellings: + - -U + value_name: enables + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -v + description: enables verbose output + provenance: + sources: + - help-text +- spellings: + - -V + description: print version number and exit + provenance: + sources: + - help-text +- spellings: + - -w + description: specifies to wait forever for a reply + provenance: + sources: + - help-text +- spellings: + - -W + value_name: time + value_kind: Required + description: specifies how long to wait for a reply + provenance: + sources: + - help-text +- spellings: + - '-4' + value_name: use + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - '-6' + value_name: use + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -i + provenance: + sources: + - help-text-synopsis +provenance: + sources: + - help-text + confidence: 0.0 +children_filled: true diff --git a/corpus/host/9.18.39/help.stderr.txt b/corpus/host/9.18.39/help.stderr.txt new file mode 100644 index 00000000..e8734c2e --- /dev/null +++ b/corpus/host/9.18.39/help.stderr.txt @@ -0,0 +1,24 @@ +host: illegal option -- - +Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] + [-R number] [-m flag] [-p port] hostname [server] + -a is equivalent to -v -t ANY + -A is like -a but omits RRSIG, NSEC, NSEC3 + -c specifies query class for non-IN data + -C compares SOA records on authoritative nameservers + -d is equivalent to -v + -l lists all hosts in a domain, using AXFR + -m set memory debugging flag (trace|record|usage) + -N changes the number of dots allowed before root lookup is done + -p specifies the port on the server to query + -r disables recursive processing + -R specifies number of retries for UDP packets + -s a SERVFAIL response should stop query + -t specifies the query type + -T enables TCP/IP mode + -U enables UDP mode + -v enables verbose output + -V print version number and exit + -w specifies to wait forever for a reply + -W specifies how long to wait for a reply + -4 use IPv4 query transport only + -6 use IPv6 query transport only diff --git a/corpus/host/9.18.39/help.txt b/corpus/host/9.18.39/help.txt new file mode 100644 index 00000000..e69de29b diff --git a/corpus/host/9.18.39/meta.toml b/corpus/host/9.18.39/meta.toml new file mode 100644 index 00000000..fddbf254 --- /dev/null +++ b/corpus/host/9.18.39/meta.toml @@ -0,0 +1,75 @@ +# `host --help` writes only its getopt complaint to stdout... no, to +# stderr: `host: illegal option -- -` (S-162's own shape, already +# stripped), then a single-dash option table with no column gap +# anywhere and no placeholder column. Every row reads its description's +# first bare word as a fabricated value before this fixture's fix. +# +# Fixed by reading each flag's real value from `host`'s own usage line +# (`Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] +# [-R number] [-m flag] [-p port] hostname [server]`), the same grammar +# and bundle logic `extract_usage_flags` already uses everywhere else: +# `-c`/`-N`/`-t`/`-W`/`-R`/`-m`/`-p` each keep the spaced placeholder +# usage names them (`class`/`ndots`/`type`/`time`/`number`/`flag`/`port`) +# and gain their real description; every bundle member +# (`-a`/`-C`/`-d`/`-l`/`-r`/`-T`/`-v`/`-V`/`-w`) is read boolean, per the +# bundle `[-aCdilrTvVw]`, and also gains its real description. +# +# Honest residual, deliberately not claimed here: `-A`, `-s`, `-U`, `-4` +# and `-6` are named in NO usage line at all (not in the bundle, not in +# their own bracket group), so the fix has no evidence for them and +# leaves them exactly as broken as before: a fabricated value guessed +# from their own description's first word, and no description recovered. +# `must_not_contain_flags` below is not the tool for stating that; it is +# recorded here in prose since the corpus contract has no "known still +# wrong" field. +# +# `-i` also appears in the tree, undescribed: the usage bundle +# `[-aCdilrTvVw]` names it but no option row documents it. Pre-existing +# behaviour (spec [M-15], usage-derived flags), not something this fix +# changed. + +[bless] +provenance = "agent" + +[tool] +name = "host" +version = "9.18.39" +platform = "ubuntu-24.04" +captured_with = "manual capture per corpus/README.md step 2 (TERM=dumb NO_COLOR=1 COLUMNS=100 LC_ALL=C.UTF-8); bind9-host 1:9.18.39-0ubuntu0.24.04.7" + +[[capture]] +argv = ["host", "--help"] +stdout = "help.txt" +stderr = "help.stderr.txt" +exit_code = 1 + +[contract] +# `mandible --doctor host` reports framework: unidentified. +expected_framework = "generic" + +[contract.must_describe] +"-a" = "is equivalent to -v -t ANY" +"-C" = "compares SOA records on authoritative nameservers" +"-c" = "specifies query class for non-IN data" +"-d" = "is equivalent to -v" +"-l" = "lists all hosts in a domain, using AXFR" +"-m" = "set memory debugging flag (trace|record|usage)" +"-N" = "changes the number of dots allowed before root lookup is done" +"-p" = "specifies the port on the server to query" +"-r" = "disables recursive processing" +"-R" = "specifies number of retries for UDP packets" +"-t" = "specifies the query type" +"-T" = "enables TCP/IP mode" +"-v" = "enables verbose output" +"-V" = "print version number and exit" +"-w" = "specifies to wait forever for a reply" +"-W" = "specifies how long to wait for a reply" + +[contract.must_value_name] +"-c" = "class" +"-N" = "ndots" +"-t" = "type" +"-W" = "time" +"-R" = "number" +"-m" = "flag" +"-p" = "port" diff --git a/corpus/lsof/4.95.0/expected.snap b/corpus/lsof/4.95.0/expected.snap index 44d1b724..44761010 100644 --- a/corpus/lsof/4.95.0/expected.snap +++ b/corpus/lsof/4.95.0/expected.snap @@ -155,6 +155,7 @@ flags: - -F value_name: f value_kind: Optional + description: '[f] select fields; -F? for help' provenance: sources: - help-text diff --git a/docs/design.md b/docs/design.md index 12f67da4..206195bf 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2983,19 +2983,26 @@ Re-measured with `xtask coverage --tools lldb-server,lldb-server-18` against the real installed binaries: `existence_fabrication_tools` reads 0 and `exist` reads 0 for both. Docs/shapes.md S-167. -**`memhog`'s whole screen ships below the five-tool bar (2026-09-13).** -Maintainer, after looking at mandible's screen for the tool: "yeah this one -needs full revision". `memhog --help` has no blank line anywhere and no -`usage:` label; every one of its five raw lines was wrong or lost before -this round (docs/shapes.md S-165, S-176, S-177). Tree-level, each shape -reads one tool, `memhog` itself; a raw-text grep over -`audit/queue-captures/*/0.std*` reads 62 tools for S-176's own row shape -and 48 for S-177's, both explicitly upper bounds on the raw text rather -than a tree-level count, since most of those rows carry a `--long` row or -a labelled multi-line synopsis that keeps the narrower gate closed. Ships -anyway: `corpus/memhog/2.0.18` promotes out of `[xfail]`, every contract -field it names now passes, and the corpus sweep and named controls stay -byte-identical. +**`memhog`'s whole screen was wrong or lost, S-177's own positional +recovery ships below the five-tool bar, S-176's own value-name rule +does not (2026-09-13).** Maintainer, after looking at mandible's screen +for the tool: "yeah this one needs full revision". `memhog --help` has +no blank line anywhere and no `usage:` label; every one of its five raw +lines was wrong or lost before this round (docs/shapes.md S-165, S-176, +S-177). S-176's own rule — a single-dash table row's real value comes +from the tool's own usage line, never a guess at the row's own text — +was first measured at 1 tool (`memhog` alone) and revised once a +full-`PATH` sweep on an earlier version of the rule found it threw away +genuine values on `host` (`-c`/`-N`/`-t`/`-W`/`-R`/`-m`/`-p`) that a +narrower usage-line reading now recovers correctly; the revised rule +reads tree-level at 5 tools (`memhog`, `host`, `kpartx`, `numastat`, +`savelog`), clearing the bar. S-177's own positional-flattening rule +stays at 1 tool, `memhog`, below the bar; a raw-text grep over +`audit/queue-captures/*/0.std*` reads 48 tools for its own row shape, an +explicit upper bound on the raw text rather than a tree-level count. +Both ship: `corpus/memhog/2.0.18` and `corpus/host/9.18.39` promote out +of (or never enter) `[xfail]`, every contract field they name passes, +and the corpus sweep and named controls stay byte-identical. ### Deferred, with the reason each is not simply undone diff --git a/docs/shapes.md b/docs/shapes.md index b2f14c7f..14ed1430 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3931,41 +3931,55 @@ entry's `tools` field and nothing else. It does not get a new entry. - id: S-176 - looks like: | -f mmap is backed by FILE - -H disable transparent hugepages -- tools: memhog + -c specifies query class for non-IN data + -a is equivalent to -v -t ANY +- tools: memhog, host, kpartx, numastat, savelog - handling: Fixed, inside a document with no `--long` row anywhere - ([`document_has_no_long_row`], the same discriminator S-157's own table - repair rests on) and at least two short-only `HelpText`-sourced flags - (`MIN_TABLE_ROWS`, the "more than a coincidence" floor). A row with no - genuine placeholder at all — no `<...>`/`[...]`, no ALL-CAPS metavar, no - glued `-Xvalue` — still reads the first bare word of its own description - as a value (`-f mmap is backed by FILE` to `-f` valued `"mmap"`), and the - real description is then lost outright, since nothing is left over once - that first word is spent. `recover_bare_word_first_description_word` - (`mandible-extract/src/help_text/sections/repair.rs`) recovers both - halves: the tool's own usage line is independent evidence of the real - value, present (`[-fFILE]` glues an uppercase run onto the letter, - naming `FILE`) or absent (`[-H]` glues nothing, naming none); without - that direct proof the flag's own guess is left alone. The row's text - past the flag letter becomes the description either way. A second, - narrower case in the same family: a row whose value is already genuine - (`memhog`'s own `-rNUM`) still loses its description the same way, with - no column gap to say where the spec ends and the description begins; - recovered by stripping exactly the already-known-correct value text off - the row's own remainder, never touching the value itself. -- fleet: Not measured with `xtask coverage` this round (build capacity was - contended; the orchestrator owns the full-`PATH` sweep). A raw-text grep - over `audit/queue-captures/*/0.std*` for a single-dash-letter row - followed by one space and a lowercase bare word reads 62 tools — an - upper bound on the raw shape, not a tree-level count, and most of those - 62 (`ffmpeg`, `ffplay`, `ffprobe`, `python3.12`, `qemu-aarch64-static`, - …) carry a `--long` row elsewhere and so never reach this repair's own - gate. Tree-level: one tool, `memhog`, verified by - `mandible-extract/src/help_text/sections/mod.rs`'s - `memhog_flags_positionals_and_root_description_all_land_correctly`. - Below the five-tool bar; shipped as a gated exception (docs/design.md - §16) alongside `corpus/memhog/2.0.18`, promoted out of `[xfail]`, - maintainer-audited ("yeah this one needs full revision"). + ([`document_has_no_long_row`]) and at least two short-only `HelpText`- + sourced flags (`MIN_TABLE_ROWS`). A row with no genuine placeholder at + all still reads the first bare word of its own description as a value + (`-f mmap is backed by FILE` to `-f` valued `"mmap"`), and the real + description is lost outright once that first word is spent. The row's + own text is never trusted to say what the value is — the tool's own + usage line is, read by the exact grammar and bundle/alternation logic + [`extract_usage_flags`] already uses everywhere else + ([`usage_derived_value_for_short`]), covering all four usage spellings: + a glued uppercase run (`[-fFILE]` → `FILE`), a spaced placeholder + (`host`'s `[-c class]` → `class`), a glued optional group (`numastat`'s + `[-s[]]` → ``, Optional), and no placeholder at all, whether + alone (`[-r]`), in an alternation (`kpartx`'s `[-a|-d|-u|-l]`), or in a + bundle (`host`'s `[-aCdilrTvVw]`) — all three read boolean. A letter the + usage line never names at all (`host`'s own `-A`/`-s`/`-U`/`-4`/`-6`) + gets no change at all: no evidence, no guess. The row's own remainder + becomes the description either way, minus one leading occurrence of the + usage value when the row's own text happens to open with it too + (`savelog`'s `-r rolldir - use rolldir...`). A row packing a second + flag by a real two-plus-space column gap (`lsof`'s own multi-column + `-T`/`-U`/`-v` summary line) is refused rather than read as one row's + description; a single-spaced mention of another flag in ordinary prose + (`kpartx`'s `-l list partitions ... added by -a`) is not mistaken for + that shape. +- fleet: Tree-level: 5 tools (`memhog`, `host`, `kpartx`, `numastat`, + `savelog`), each verified by its own unit test in + `mandible-extract/src/help_text/sections/repair.rs` (`host` also by + `corpus/host/9.18.39`, promoted straight in — never `[xfail]`). Clears + the five-tool bar (AGENTS.md §3.1) on that count; not swept fleet-wide + with `xtask coverage` this round (build capacity was contended, and the + orchestrator owns the full-`PATH` sweep). A raw-text grep over + `audit/queue-captures/*/0.std*` for a single-dash-letter row followed by + one space and a lowercase bare word reads 62 tools — an explicit upper + bound on the raw shape, not a tree-level count; most of those 62 + (`ffmpeg`, `ffplay`, `ffprobe`, `python3.12`, `qemu-aarch64-static`, …) + carry a `--long` row elsewhere and never reach this repair's own gate. + Honest residual: `host`'s own `-A`, `-s`, `-U`, `-4`, `-6`, named in no + usage line at all, stay exactly as wrong as before — 5 rows on 1 tool, + recorded in `corpus/host/9.18.39/meta.toml` rather than claimed fixed. + Zero-loss checked against `git diff`'s own field-level list on every + corpus fixture this round: no `value_name` went from something real to + nothing anywhere except `memhog`'s own already-fenced S-172 case (S-177 + gained, `lsof`'s own `-F` gained a description with its value + untouched), and `Xvfb`/`mksquashfs`/`sqfstar` stayed byte-identical + (`pty_screenshot.py`, before/after diff empty). ### S-177: a usage synopsis's lowercase operand tail, unlabelled and nested diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 20bc165b..8866d4a5 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -396,14 +396,14 @@ pub(super) fn repair_usage_attested_single_dash_long(flags: &mut [Entity], usage /// A single-dash table row with no genuine placeholder (`memhog`'s `-f /// mmap is backed by FILE`) reads its description's first word as a -/// fabricated value, losing the description. The usage line is -/// independent evidence: present (`[-fFILE]`) or absent (`[-H]`); no -/// proof, no change. A narrower case: an already-genuine value (`-rNUM`, -/// S-172) still loses its description, recovered by stripping the known -/// value off the row's remainder. Scoped, like S-157, to a document with -/// no `--long` row ([`document_has_no_long_row`]); the floor is a flag -/// count ([`MIN_TABLE_ROWS`]), not [`single_dash_long_table`]'s -/// multi-character name. See docs/shapes.md S-176. +/// fabricated value; the row's own text is never trusted to say what a +/// value is. The tool's own usage line is +/// ([`usage_derived_value_for_short`], same grammar and bundle logic +/// [`extract_usage_flags`] already uses) or it is left alone — `host`'s +/// own `-A`/`-s`/`-U`/`-4`/`-6`, named in no usage line at all, are +/// exactly this refusal. Scoped, like S-157, to a document with no +/// `--long` row ([`document_has_no_long_row`]); the floor is a flag +/// count ([`MIN_TABLE_ROWS`]). See docs/shapes.md S-176. pub(super) fn recover_bare_word_first_description_word( flags: &mut [Entity], usage_lines: &[String], @@ -424,7 +424,6 @@ pub(super) fn recover_bare_word_first_description_word( if short_only_help_text_flags < MIN_TABLE_ROWS { return; } - let dash_separator_document = document_uses_dash_separator_convention(lines); for flag in flags.iter_mut() { if !flag.provenance.sources.contains(&Source::HelpText) || flag.provenance.sources.contains(&Source::HelpTextSynopsis) @@ -435,6 +434,13 @@ pub(super) fn recover_bare_word_first_description_word( continue; } let Some(short) = flag.short() else { continue }; + // No evidence, no change: a letter the usage line never spells + // at all keeps today's row-only guess rather than being assumed + // boolean. + let Some((usage_value, usage_kind)) = usage_derived_value_for_short(usage_lines, short) + else { + continue; + }; let prefix = format!("-{short}"); // Exactly one candidate row, never the first of several: `lsof` // documents `-T` twice, once in a packed multi-column summary @@ -451,110 +457,69 @@ pub(super) fn recover_bare_word_first_description_word( continue; } let after_letter = &row.trim_start()[prefix.len()..]; - if after_letter - .split_whitespace() - .any(|w| w.len() > 1 && w.starts_with('-') && w[1..].starts_with(char::is_alphabetic)) - { - continue; - } - // A ` - ` (space-dash-space) run right after the row's own value - // token, in a document whose other option rows show the same - // separator (`savelog`'s `-m mode - chmod...`, every row), - // is the document's own description boundary, not a coincidence: - // the value is genuine even though its own row's gap collapsed to - // one space (`-r rolldir - use rolldir...`, `rolldir` too long to - // leave room for a wider gap before the dash). Takes priority - // over the bare-word guess below, never touching the value. - if flag.value_kind == ValueKind::Required && dash_separator_document { - if let Some(dash_idx) = find_dash_separator(row) { - let (_, desc) = split_at_dash(row, dash_idx); - if !desc.trim().is_empty() { - flag.description = non_empty_text(desc.trim()); - continue; - } - } - } - // The captured "value" must itself be shaped like a bare - // description word, never a genuine placeholder: one run of - // nothing but lowercase ASCII letters. - let is_bare_word_guess = flag.value_kind == ValueKind::Required - && flag - .value_name - .as_deref() - .is_some_and(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_lowercase())); - if is_bare_word_guess { - // The fabrication shape is always spaced (`-f mmap...`): a - // glued tail (`-rNUM`) is the other, already-correct case - // below, never this one. - if !after_letter.starts_with(char::is_whitespace) { - continue; - } - let desc = after_letter.trim(); - if desc.is_empty() { + // A second flag token *packed onto the same line by a real column + // gap* (`lsof`'s `-T disable TCP/TPI info -U select Unix + // socket`, two-plus spaces before `-U`) is refused — that row + // documents more than one flag and taking the whole remainder + // would read a neighbour's row into this one's description. A + // single-spaced dash-word is ordinary prose mentioning another + // flag (`kpartx`'s own `-l list partitions ... added by -a`) and + // must not trip this guard. + if let Some(gap) = find_multi_space_gap(after_letter) { + if after_letter[gap..].trim_start().starts_with('-') { continue; } - let Some(usage_value) = usage_lines - .iter() - .find_map(|u| usage_line_glued_short_value(u, short)) - else { - continue; // no independent evidence; leave today's guess alone - }; - match usage_value { - Some(value) => { - flag.value_name = Some(value); - flag.value_kind = ValueKind::Required; - } - None => { - flag.value_name = None; - flag.value_kind = ValueKind::None; + } + // The row's own remainder is the description, whole, unless it + // happens to open with the usage-named value as its own leading + // word (`savelog`'s `-r rolldir - use rolldir...`: `rolldir` is + // both the value and the row's own first token) — then only that + // leading occurrence, and an immediately following ` - ` + // separator, are stripped off it. `host`'s `-c specifies query + // class for non-IN data` has no such overlap (`class`, the real + // value, never opens the row), so the whole sentence stays the + // description exactly as it reads. + let trimmed = after_letter.trim_start(); + let desc_source = match usage_value.as_deref() { + Some(v) if !v.is_empty() => match trimmed.strip_prefix(v) { + Some(rest) if rest.is_empty() || rest.starts_with(char::is_whitespace) => { + let rest = rest.trim_start(); + rest.strip_prefix("- ").unwrap_or(rest) } - } - flag.description = non_empty_text(desc); - } else if flag.value_kind == ValueKind::Required { - let Some(value) = flag.value_name.as_deref() else { - continue; - }; - let Some(rest) = after_letter.strip_prefix(value) else { - continue; - }; - let desc = rest.trim(); - if desc.is_empty() { - continue; - } - flag.description = non_empty_text(desc); + _ => trimmed, + }, + _ => trimmed, + }; + let desc = desc_source.trim(); + if desc.is_empty() { + continue; } + flag.value_name = usage_value; + flag.value_kind = usage_kind; + flag.description = non_empty_text(desc); } } -/// One usage line's own standalone bracketed token for `-short`, read as -/// glued-value evidence: `Some(Some(value))` when an uppercase run is -/// glued directly onto the letter with no space (`[-fFILE]` -> `"FILE"`), -/// `Some(None)` when the token is the bare letter alone (`[-H]`), `None` -/// when this line attests nothing about the letter at all. Boundary rule -/// mirrors [`usage_line_has_standalone_token`]: whitespace or a bracket on -/// both sides, so a longer spelling sharing the same prefix is never -/// mistaken for this one. See docs/shapes.md S-176. -fn usage_line_glued_short_value(line: &str, short: char) -> Option> { - let needle = format!("-{short}"); - let is_boundary = |c: char| c.is_whitespace() || c == '[' || c == ']'; - let mut start = 0usize; - while let Some(rel) = line[start..].find(&needle) { - let idx = start + rel; - let before_ok = line[..idx].chars().next_back().is_none_or(is_boundary); - if before_ok { - let after = &line[idx + needle.len()..]; - let tail_end = after.find(is_boundary).unwrap_or(after.len()); - let tail = &after[..tail_end]; - if tail.is_empty() { - return Some(None); - } - if tail.chars().all(|c| c.is_ascii_uppercase()) { - return Some(Some(tail.to_string())); - } - } - start = idx + 1; - } - None +/// The value [`extract_usage_flags`] itself would assign `short`, read +/// straight off the usage line by the same grammar (a spaced placeholder, +/// `[-c class]`; a glued optional group, `[-s[]]`; a glued +/// uppercase run, `[-fFILE]`) and the same bundle/alternation logic +/// ([`push_usage_token`]) every other usage-derived flag already goes +/// through — never a second, narrower reading of the usage line built +/// just for this repair. `Some((None, ValueKind::None))` when the usage +/// line spells the letter with no placeholder at all, alone, in an +/// alternation, or in a bundle (`kpartx`'s `[-a|-d|-u|-l]`, `host`'s +/// `[-aCdilrTvVw]`). `None` when the usage line never names this letter, +/// so the caller leaves the row untouched rather than assuming boolean. +/// See docs/shapes.md S-176. +fn usage_derived_value_for_short( + usage_lines: &[String], + short: char, +) -> Option<(Option, ValueKind)> { + extract_usage_flags(usage_lines) + .into_iter() + .find(|f| f.short() == Some(short) && f.long().is_none()) + .map(|f| (f.value_name, f.value_kind)) } /// A dash-prefixed usage-line word that normally reads as the generic @@ -702,22 +667,6 @@ fn document_has_no_long_row(lines: &[&str]) -> bool { true } -/// True when at least [`MIN_TABLE_ROWS`] of the document's own option -/// rows show a ` - ` (space-dash-space) description separator -/// ([`find_dash_separator`]) — `savelog`'s own convention, every row -/// from `-m mode - chmod...` to `-r rolldir - use rolldir...`. Evidence -/// that the document's own boundary is this literal separator, not the -/// width of the gap before it, so a row whose gap happens to collapse to -/// one space (a long value crowding out the padding) is not mistaken for -/// a fabricated placeholder. See docs/shapes.md S-176. -fn document_uses_dash_separator_convention(lines: &[&str]) -> bool { - lines - .iter() - .filter(|l| looks_like_flag_start(l.trim_start()) && find_dash_separator(l).is_some()) - .count() - >= MIN_TABLE_ROWS -} - /// True when `rest` (a row's own text, dash already stripped) opens with /// a name at least two characters long, every one of them a lowercase /// letter, digit, hyphen or underscore — a spelling no reasonable @@ -1203,16 +1152,17 @@ mod tests { } /// `savelog --help`'s own bytes, byte-exact (minus the trailing rows - /// this test doesn't need): every option row uses a ` - ` separator, - /// most with a wide enough gap that the ordinary column-gap grammar - /// already reads them (`-m`, `-u`, `-g`, `-c` all keep their real - /// descriptions untouched by this repair). `-r rolldir`'s own gap - /// collapses to one space because `rolldir` is long, which used to - /// read `rolldir` as a fabricated placeholder guess and throw the - /// real value away recovering the description. The document's own - /// separator, corroborated by its sibling rows, says `rolldir` was - /// always genuine — value and description both survive. See - /// docs/shapes.md S-176 and audit/queue-captures/savelog/0.stdout. + /// this test doesn't need): `-r rolldir`'s usage placeholder is + /// `rolldir` itself, a spaced usage-line value + /// ([`usage_derived_value_for_short`]'s own case), and `rolldir` also + /// happens to be the row's own leading word (`-r rolldir - use + /// rolldir...`) — the row's gap before its own ` - ` separator + /// collapsed to one space because `rolldir` is long, which used to + /// read it as a fabricated placeholder guess and throw the real + /// value away recovering the description. Both survive: value from + /// the usage line, description from the row's own remainder past + /// that leading occurrence. See docs/shapes.md S-176 and + /// audit/queue-captures/savelog/0.stdout. #[test] fn savelogs_own_dash_separator_keeps_a_narrow_gapped_value_and_recovers_its_description() { let raw = concat!( @@ -1242,6 +1192,136 @@ mod tests { ); } + /// `host --help`'s own bytes, byte-exact: a single-dash table with no + /// column gap at all, where every row used to read its description's + /// first word as a fabricated value. `-c`/`-N`/`-t`/`-W` each get + /// their real value (`class`/`ndots`/`type`/`time`) from `host`'s own + /// usage line's spaced placeholder; `-a` is a member of the usage + /// line's own bundle `[-aCdilrTvVw]` and is read boolean, its real + /// description recovered with no value fabricated. `-A`, named in no + /// usage line at all, is the deliberate residual this rule leaves + /// alone rather than guess at. See docs/shapes.md S-176 and + /// corpus/host/9.18.39. + #[test] + fn hosts_own_usage_line_supplies_every_valued_flags_real_name() { + let raw = concat!( + "Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time]\n", + " [-R number] [-m flag] [-p port] hostname [server]\n", + " -a is equivalent to -v -t ANY\n", + " -A is like -a but omits RRSIG, NSEC, NSEC3\n", + " -c specifies query class for non-IN data\n", + " -N changes the number of dots allowed before root lookup is done\n", + " -t specifies the query type\n", + " -W specifies how long to wait for a reply\n", + ); + let parsed = parse_named(raw, "host"); + let short = |c: char| { + parsed + .flags + .iter() + .find(|f| f.short() == Some(c)) + .unwrap_or_else(|| panic!("no -{c} in {:?}", parsed.flags)) + }; + for (letter, value) in [('c', "class"), ('N', "ndots"), ('t', "type"), ('W', "time")] { + let f = short(letter); + assert_eq!( + f.value_name.as_deref(), + Some(value), + "-{letter} must keep the usage line's own value name" + ); + assert_eq!(f.value_kind, ValueKind::Required); + assert!( + f.description.is_some(), + "-{letter} must recover its real description too" + ); + } + let a = short('a'); + assert_eq!(a.value_name, None, "-a is boolean per the usage bundle"); + assert_eq!(a.value_kind, ValueKind::None); + assert_eq!( + a.description.as_ref().map(Text::as_str), + Some("is equivalent to -v -t ANY") + ); + // `-A` is named in no usage line at all (not in the bundle, not + // in its own bracket group): no evidence, no change. It keeps + // today's fabricated guess rather than being assumed boolean. + let big_a = short('A'); + assert_eq!(big_a.value_name.as_deref(), Some("is")); + assert!(big_a.description.is_none()); + } + + /// `kpartx --help`'s own bytes, byte-exact: every flag is boolean. + /// `-f`/`-g`/`-p`/`-r`/`-v`, each its own bracketed usage token + /// (`[-r]`), already lost their fabricated values to the row-only + /// repair; `-a`/`-d`/`-u`/`-l`, members of the usage line's own + /// alternation `[-a|-d|-u|-l]`, did not — the alternation shape is + /// this rule's own addition. `-l`'s own row, "list partitions ... + /// added by -a", is prose merely mentioning another flag and must + /// not trip the packed-multi-column-row guard the way a real + /// two-plus-space gap would. See docs/shapes.md S-176 and + /// corpus/kpartx (unfixtured; captured live, `kpartx` prints its own + /// usage with no arguments). + #[test] + fn kpartxs_alternation_members_are_read_boolean_not_valued() { + let raw = concat!( + "Usage:\n", + " kpartx [-a|-d|-u|-l] [-r] [-p] [-f] [-g] [-s|-n] [-v] wholedisk\n", + "\t-a add partition devmappings\n", + "\t-r devmappings will be readonly\n", + "\t-d del partition devmappings\n", + "\t-u update partition devmappings\n", + "\t-l list partitions devmappings that would be added by -a\n", + "\t-v verbose\n", + ); + let parsed = parse_named(raw, "kpartx"); + for letter in ['a', 'd', 'u', 'l', 'r', 'v'] { + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some(letter)) + .unwrap_or_else(|| panic!("no -{letter} in {:?}", parsed.flags)); + assert_eq!(f.value_name, None, "-{letter} must be read boolean"); + assert_eq!(f.value_kind, ValueKind::None); + assert!( + f.description.is_some(), + "-{letter} must still recover its real description" + ); + } + } + + /// `numastat --help`'s own bytes, byte-exact: `-s[]` is a + /// glued optional group on the usage line, the third of the rule's + /// four usage spellings (a bare uppercase run, a spaced placeholder, + /// a glued optional group, no placeholder at all). `-p |` + /// is a spaced placeholder carrying its own alternation. See + /// docs/shapes.md S-176. + #[test] + fn numastats_glued_optional_group_keeps_its_own_bracket_value() { + let raw = concat!( + "Usage: numastat [-c] [-p |] [-s[]] [-v]\n", + "-c to minimize column widths\n", + "-p | to show process info\n", + "-s[] to sort data by total column or \n", + "-v to make some reports more verbose\n", + ); + let parsed = parse_named(raw, "numastat"); + let s = parsed + .flags + .iter() + .find(|f| f.short() == Some('s')) + .expect("no -s in numastat's flags"); + assert_eq!(s.value_kind, ValueKind::Optional); + assert!(s.value_name.as_deref().is_some_and(|v| v.contains("node"))); + assert!(s.description.is_some()); + let p = parsed + .flags + .iter() + .find(|f| f.short() == Some('p')) + .expect("no -p in numastat's flags"); + assert_eq!(p.value_kind, ValueKind::Required); + assert!(p.description.is_some()); + } + /// A spaced value is indistinguishable from a glued one once stored; /// the raw text is what decides. #[test] From 1b67aaacb1a5bd19d38e21252cf532a123346c6c Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 23:26:23 +0400 Subject: [PATCH 5/5] [S-176] strip a row's bracketed repeat of its own usage value lsof's `-F [f] select fields; -F? for help` printed `[f]` twice: once as the flag's Optional value, once again opening the description the same rule had just recovered. Bare, bracketed and angled spellings of the leading value are now stripped alike. Co-Authored-By: Claude Fable 5.1 --- corpus/lsof/4.95.0/expected.snap | 2 +- docs/shapes.md | 7 +- .../src/help_text/sections/repair.rs | 86 ++++++++++++++++--- 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/corpus/lsof/4.95.0/expected.snap b/corpus/lsof/4.95.0/expected.snap index 44761010..fb3291c7 100644 --- a/corpus/lsof/4.95.0/expected.snap +++ b/corpus/lsof/4.95.0/expected.snap @@ -155,7 +155,7 @@ flags: - -F value_name: f value_kind: Optional - description: '[f] select fields; -F? for help' + description: select fields; -F? for help provenance: sources: - help-text diff --git a/docs/shapes.md b/docs/shapes.md index 14ed1430..bcc165a4 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3952,8 +3952,11 @@ entry's `tools` field and nothing else. It does not get a new entry. usage line never names at all (`host`'s own `-A`/`-s`/`-U`/`-4`/`-6`) gets no change at all: no evidence, no guess. The row's own remainder becomes the description either way, minus one leading occurrence of the - usage value when the row's own text happens to open with it too - (`savelog`'s `-r rolldir - use rolldir...`). A row packing a second + usage value when the row's own text happens to open with it too, in + whichever spelling that row uses for it — bare (`savelog`'s `-r rolldir + - use rolldir...`), bracketed (`lsof`'s `-F [f] select fields; -F? for + help`, whose Optional `f` would otherwise be printed twice) or angled + ([`strip_leading_value_spelling`]). A row packing a second flag by a real two-plus-space column gap (`lsof`'s own multi-column `-T`/`-U`/`-v` summary line) is refused rather than read as one row's description; a single-spaced mention of another flag in ordinary prose diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 8866d4a5..b49358f6 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -472,22 +472,17 @@ pub(super) fn recover_bare_word_first_description_word( } // The row's own remainder is the description, whole, unless it // happens to open with the usage-named value as its own leading - // word (`savelog`'s `-r rolldir - use rolldir...`: `rolldir` is - // both the value and the row's own first token) — then only that + // token (`savelog`'s `-r rolldir - use rolldir...`: `rolldir` is + // both the value and the row's own first word) — then only that // leading occurrence, and an immediately following ` - ` - // separator, are stripped off it. `host`'s `-c specifies query - // class for non-IN data` has no such overlap (`class`, the real - // value, never opens the row), so the whole sentence stays the - // description exactly as it reads. + // separator, are stripped off it + // ([`strip_leading_value_spelling`]). `host`'s `-c specifies + // query class for non-IN data` has no such overlap (`class`, the + // real value, never opens the row), so the whole sentence stays + // the description exactly as it reads. let trimmed = after_letter.trim_start(); let desc_source = match usage_value.as_deref() { - Some(v) if !v.is_empty() => match trimmed.strip_prefix(v) { - Some(rest) if rest.is_empty() || rest.starts_with(char::is_whitespace) => { - let rest = rest.trim_start(); - rest.strip_prefix("- ").unwrap_or(rest) - } - _ => trimmed, - }, + Some(v) if !v.is_empty() => strip_leading_value_spelling(trimmed, v).unwrap_or(trimmed), _ => trimmed, }; let desc = desc_source.trim(); @@ -500,6 +495,35 @@ pub(super) fn recover_bare_word_first_description_word( } } +/// The row's own leading occurrence of the usage-named value, stripped +/// off the row's remainder along with an immediately following ` - ` +/// separator, in whichever spelling the row itself uses for it: bare +/// (`savelog`'s `-r rolldir - use rolldir instead of .`), bracketed +/// (`lsof`'s `-F [f] select fields; -F? for help`, whose value is +/// already Optional and would otherwise be printed twice) or angled +/// (``). `None` when the row does not open with the value at all +/// (`host`'s `-c specifies query class for non-IN data`), and the whole +/// remainder is the description. The token must end at whitespace or at +/// the end of the row, so a longer word merely starting with the value's +/// own letters is never cut. See docs/shapes.md S-176. +fn strip_leading_value_spelling<'a>(trimmed: &'a str, value: &str) -> Option<&'a str> { + for spelling in [ + value.to_string(), + format!("[{value}]"), + format!("<{value}>"), + ] { + let Some(rest) = trimmed.strip_prefix(spelling.as_str()) else { + continue; + }; + if !rest.is_empty() && !rest.starts_with(char::is_whitespace) { + continue; + } + let rest = rest.trim_start(); + return Some(rest.strip_prefix("- ").unwrap_or(rest)); + } + None +} + /// The value [`extract_usage_flags`] itself would assign `short`, read /// straight off the usage line by the same grammar (a spaced placeholder, /// `[-c class]`; a glued optional group, `[-s[]]`; a glued @@ -1289,6 +1313,42 @@ mod tests { } } + /// `lsof -h`'s own bytes, byte-exact (the rows this test needs): + /// `-F`'s usage placeholder is the optional `[f]`, and the row's own + /// remainder opens with that same `[f]` — the description must be + /// the sentence alone, never the placeholder printed a second time + /// beside the value the flag already carries. The packed + /// multi-column `-T`/`-U`/`-v` summary line is here too: `-T` has + /// two candidate rows and must keep the real value `fqs` from its + /// own row, untouched by this repair. See docs/shapes.md S-176 and + /// corpus/lsof/4.95.0. + #[test] + fn lsofs_bracketed_optional_value_is_not_printed_twice_in_its_description() { + let raw = concat!( + "lsof 4.95.0\n", + " usage: [-?abhKlnNoOPRtUvVX] [+|-c c] [+|-d s] [+D D] [+|-E] [+|-e s] [+|-f[gG]]\n", + " [-F [f]] [-g [s]] [-i [i]] [+|-L [l]] [+m [m]] [+|-M] [-o [o]] [-p s]\n", + " [+|-r [t]] [-s [p:s]] [-S [t]] [-T [t]] [-u s] [+|-w] [-x [fl]] [--] [names]\n", + " -T disable TCP/TPI info -U select Unix socket -v list version info\n", + " -F [f] select fields; -F? for help \n", + " -T fqs TCP/TPI Fl,Q,St (s) info\n", + " -g [s] exclude(^)|select and print process group IDs\n", + ); + let parsed = parse_named(raw, "lsof"); + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some('F')) + .unwrap_or_else(|| panic!("no -F in {:?}", parsed.flags)); + assert_eq!(f.value_name.as_deref(), Some("f")); + assert_eq!(f.value_kind, ValueKind::Optional); + assert_eq!( + f.description.as_ref().map(Text::as_str), + Some("select fields; -F? for help"), + "-F's own bracketed placeholder must not open its description" + ); + } + /// `numastat --help`'s own bytes, byte-exact: `-s[]` is a /// glued optional group on the usage line, the third of the rule's /// four usage spellings (a bare uppercase run, a spaced placeholder,