diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e05ad6b..a61c3623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ once it reaches a published 0.1.0 release. ## [Unreleased] +### Fixed + +- [S-176] A single-dash table row with no placeholder column now reads its flag's real value off the tool's own usage line instead of guessing it from the description's first word (`mandible host`, `mandible kpartx`, `mandible numastat`, `mandible savelog`, `mandible memhog`); `mandible memhog` also gains its `size`/`policy`/`nodeset` positionals (S-177) instead of a swallowed root description. + ## [0.8.0] - 2026-09-14 ### Changed diff --git a/corpus/dbiprof/1.643/expected.snap b/corpus/dbiprof/1.643/expected.snap index a5561d55..080ca49e 100644 --- a/corpus/dbiprof/1.643/expected.snap +++ b/corpus/dbiprof/1.643/expected.snap @@ -1,6 +1,11 @@ name: dbiprof usage: - dbiprof [options] [files] +positionals: +- name: files + provenance: + sources: + - help-text flags: - spellings: - -number diff --git a/corpus/host/9.18.39/expected.snap b/corpus/host/9.18.39/expected.snap new file mode 100644 index 00000000..37c66423 --- /dev/null +++ b/corpus/host/9.18.39/expected.snap @@ -0,0 +1,159 @@ +name: host +usage: +- 'Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] [-R number] [-m flag] [-p port] hostname [server]' +flags: +- spellings: + - -a + description: is equivalent to -v -t ANY + provenance: + sources: + - help-text +- spellings: + - -A + value_name: is + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -c + value_name: class + value_kind: Required + description: specifies query class for non-IN data + provenance: + sources: + - help-text +- spellings: + - -C + description: compares SOA records on authoritative nameservers + provenance: + sources: + - help-text +- spellings: + - -d + description: is equivalent to -v + provenance: + sources: + - help-text +- spellings: + - -l + description: lists all hosts in a domain, using AXFR + provenance: + sources: + - help-text +- spellings: + - -m + value_name: flag + value_kind: Required + description: set memory debugging flag (trace|record|usage) + provenance: + sources: + - help-text +- spellings: + - -N + value_name: ndots + value_kind: Required + description: changes the number of dots allowed before root lookup is done + provenance: + sources: + - help-text +- spellings: + - -p + value_name: port + value_kind: Required + description: specifies the port on the server to query + provenance: + sources: + - help-text +- spellings: + - -r + description: disables recursive processing + provenance: + sources: + - help-text +- spellings: + - -R + value_name: number + value_kind: Required + description: specifies number of retries for UDP packets + provenance: + sources: + - help-text +- spellings: + - -s + value_name: a + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -t + value_name: type + value_kind: Required + description: specifies the query type + provenance: + sources: + - help-text +- spellings: + - -T + description: enables TCP/IP mode + provenance: + sources: + - help-text +- spellings: + - -U + value_name: enables + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -v + description: enables verbose output + provenance: + sources: + - help-text +- spellings: + - -V + description: print version number and exit + provenance: + sources: + - help-text +- spellings: + - -w + description: specifies to wait forever for a reply + provenance: + sources: + - help-text +- spellings: + - -W + value_name: time + value_kind: Required + description: specifies how long to wait for a reply + provenance: + sources: + - help-text +- spellings: + - '-4' + value_name: use + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - '-6' + value_name: use + value_kind: Required + provenance: + sources: + - help-text +- spellings: + - -i + provenance: + sources: + - help-text-synopsis +provenance: + sources: + - help-text + confidence: 0.0 +children_filled: true diff --git a/corpus/host/9.18.39/help.stderr.txt b/corpus/host/9.18.39/help.stderr.txt new file mode 100644 index 00000000..e8734c2e --- /dev/null +++ b/corpus/host/9.18.39/help.stderr.txt @@ -0,0 +1,24 @@ +host: illegal option -- - +Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] + [-R number] [-m flag] [-p port] hostname [server] + -a is equivalent to -v -t ANY + -A is like -a but omits RRSIG, NSEC, NSEC3 + -c specifies query class for non-IN data + -C compares SOA records on authoritative nameservers + -d is equivalent to -v + -l lists all hosts in a domain, using AXFR + -m set memory debugging flag (trace|record|usage) + -N changes the number of dots allowed before root lookup is done + -p specifies the port on the server to query + -r disables recursive processing + -R specifies number of retries for UDP packets + -s a SERVFAIL response should stop query + -t specifies the query type + -T enables TCP/IP mode + -U enables UDP mode + -v enables verbose output + -V print version number and exit + -w specifies to wait forever for a reply + -W specifies how long to wait for a reply + -4 use IPv4 query transport only + -6 use IPv6 query transport only diff --git a/corpus/host/9.18.39/help.txt b/corpus/host/9.18.39/help.txt new file mode 100644 index 00000000..e69de29b diff --git a/corpus/host/9.18.39/meta.toml b/corpus/host/9.18.39/meta.toml new file mode 100644 index 00000000..fddbf254 --- /dev/null +++ b/corpus/host/9.18.39/meta.toml @@ -0,0 +1,75 @@ +# `host --help` writes only its getopt complaint to stdout... no, to +# stderr: `host: illegal option -- -` (S-162's own shape, already +# stripped), then a single-dash option table with no column gap +# anywhere and no placeholder column. Every row reads its description's +# first bare word as a fabricated value before this fixture's fix. +# +# Fixed by reading each flag's real value from `host`'s own usage line +# (`Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time] +# [-R number] [-m flag] [-p port] hostname [server]`), the same grammar +# and bundle logic `extract_usage_flags` already uses everywhere else: +# `-c`/`-N`/`-t`/`-W`/`-R`/`-m`/`-p` each keep the spaced placeholder +# usage names them (`class`/`ndots`/`type`/`time`/`number`/`flag`/`port`) +# and gain their real description; every bundle member +# (`-a`/`-C`/`-d`/`-l`/`-r`/`-T`/`-v`/`-V`/`-w`) is read boolean, per the +# bundle `[-aCdilrTvVw]`, and also gains its real description. +# +# Honest residual, deliberately not claimed here: `-A`, `-s`, `-U`, `-4` +# and `-6` are named in NO usage line at all (not in the bundle, not in +# their own bracket group), so the fix has no evidence for them and +# leaves them exactly as broken as before: a fabricated value guessed +# from their own description's first word, and no description recovered. +# `must_not_contain_flags` below is not the tool for stating that; it is +# recorded here in prose since the corpus contract has no "known still +# wrong" field. +# +# `-i` also appears in the tree, undescribed: the usage bundle +# `[-aCdilrTvVw]` names it but no option row documents it. Pre-existing +# behaviour (spec [M-15], usage-derived flags), not something this fix +# changed. + +[bless] +provenance = "agent" + +[tool] +name = "host" +version = "9.18.39" +platform = "ubuntu-24.04" +captured_with = "manual capture per corpus/README.md step 2 (TERM=dumb NO_COLOR=1 COLUMNS=100 LC_ALL=C.UTF-8); bind9-host 1:9.18.39-0ubuntu0.24.04.7" + +[[capture]] +argv = ["host", "--help"] +stdout = "help.txt" +stderr = "help.stderr.txt" +exit_code = 1 + +[contract] +# `mandible --doctor host` reports framework: unidentified. +expected_framework = "generic" + +[contract.must_describe] +"-a" = "is equivalent to -v -t ANY" +"-C" = "compares SOA records on authoritative nameservers" +"-c" = "specifies query class for non-IN data" +"-d" = "is equivalent to -v" +"-l" = "lists all hosts in a domain, using AXFR" +"-m" = "set memory debugging flag (trace|record|usage)" +"-N" = "changes the number of dots allowed before root lookup is done" +"-p" = "specifies the port on the server to query" +"-r" = "disables recursive processing" +"-R" = "specifies number of retries for UDP packets" +"-t" = "specifies the query type" +"-T" = "enables TCP/IP mode" +"-v" = "enables verbose output" +"-V" = "print version number and exit" +"-w" = "specifies to wait forever for a reply" +"-W" = "specifies how long to wait for a reply" + +[contract.must_value_name] +"-c" = "class" +"-N" = "ndots" +"-t" = "type" +"-W" = "time" +"-R" = "number" +"-m" = "flag" +"-p" = "port" diff --git a/corpus/lsof/4.95.0/expected.snap b/corpus/lsof/4.95.0/expected.snap index e526efb7..fb3291c7 100644 --- a/corpus/lsof/4.95.0/expected.snap +++ b/corpus/lsof/4.95.0/expected.snap @@ -1,5 +1,8 @@ name: lsof -description: 'lsof: -e not followed by a file system path: "lp" lsof 4.95.0 Defaults in parentheses; comma-separated set (s) items; dash-separated ranges. Anyone can list all files; /dev warnings disabled; kernel ID check disabled.' +description: |- + lsof: -e not followed by a file system path: "lp" lsof 4.95.0 Defaults in parentheses; comma-separated set (s) items; dash-separated ranges. + + Anyone can list all files; /dev warnings disabled; kernel ID check disabled. usage: - 'usage: [-?abhKlnNoOPRtUvVX] [+|-c c] [+|-d s] [+D D] [+|-E] [+|-e s] [+|-f[gG]] [-F [f]] [-g [s]] [-i [i]] [+|-L [l]] [+m [m]] [+|-M] [-o [o]] [-p s] [+|-r [t]] [-s [p:s]] [-S [t]] [-T [t]] [-u s] [+|-w] [-x [fl]] [--] [names]' flags: @@ -152,6 +155,7 @@ flags: - -F value_name: f value_kind: Optional + description: select fields; -F? for help provenance: sources: - help-text diff --git a/corpus/memhog/2.0.18/expected.snap b/corpus/memhog/2.0.18/expected.snap new file mode 100644 index 00000000..c2e424bd --- /dev/null +++ b/corpus/memhog/2.0.18/expected.snap @@ -0,0 +1,46 @@ +name: memhog +description: 'Policies: preferred-many local interleave membind preferred default' +usage: +- memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]] +positionals: +- name: size + required: true + provenance: + sources: + - help-text +- name: policy + provenance: + sources: + - help-text +- name: nodeset + provenance: + sources: + - help-text +flags: +- spellings: + - -f + value_name: FILE + value_kind: Required + description: mmap is backed by FILE + provenance: + sources: + - help-text +- spellings: + - -r + value_name: NUM + value_kind: Required + description: repeat memset NUM times + provenance: + sources: + - help-text +- spellings: + - -H + description: disable transparent hugepages + provenance: + sources: + - help-text +provenance: + sources: + - help-text + confidence: 0.0 +children_filled: true diff --git a/corpus/memhog/2.0.18/meta.toml b/corpus/memhog/2.0.18/meta.toml index 22a76f3e..bc660914 100644 --- a/corpus/memhog/2.0.18/meta.toml +++ b/corpus/memhog/2.0.18/meta.toml @@ -1,44 +1,10 @@ -# No memhog fixture existed in this corpus before this one. `memhog` ships -# in the `numactl` package (2.0.18-1ubuntu0.24.04.1 on this build) and its -# `--help` exits 1 with everything on stdout. -# -# Maintainer, after looking at mandible's screen for this tool: "yeah this -# one needs full revision". The whole screen is the contract gap; every one -# of the raw capture's five lines is wrong or lost in the parsed tree. Row -# by row, checked against `help.txt`: -# -# 1. `memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]` -# Kept verbatim in `usage`. But the positionals it documents — -# required `size[kmg]`, optional `policy` and `nodeset` — are not -# extracted at all; the tree has no `positionals` field. -# 2. `-f mmap is backed by FILE` -# `-f` is invented a value name `mmap` — the first word of its own -# description, misread as a placeholder — and carries no description. -# The real description, "mmap is backed by FILE", is not attached to -# `-f` anywhere; instead this whole line is swallowed, concatenated -# with lines 3-5, into the root node's own `description` field. -# 3. `-rNUM repeat memset NUM times` -# `-r`'s value name `NUM` is recovered correctly (the one row of the -# five with no invention), but its description "repeat memset NUM -# times" is lost the same way line 2's is: swallowed into the root -# node's `description` blob instead of attached to `-r`. -# 4. `-H disable transparent hugepages` -# Same invention as line 2: `-H` is a boolean flag with no value in -# the tool, but the parser gives it a fabricated value name `disable` -# — the first word of its own description — and no description of its -# own; the real text is swallowed into the root blob again. -# 5. `Policies: preferred-many local interleave membind preferred default` -# This line enumerates the values the `policy` positional accepts. It -# is not attached anywhere structured — there is no `policy` positional -# to attach it to, since line 1's positional extraction already failed -# — and instead becomes the tail of the same root-level `description` -# blob lines 2-4 were swallowed into. -# -# Net effect: the root node's `description` field is a single run-on -# sentence built by concatenating lines 2 through 5, and every flag that -# should have carried a piece of that text carries a fabricated value name -# instead. Confidence on this tree is 0.0 (`cargo run -p xtask -- corpus -# --show memhog`), the lowest this build reports. +# `memhog` ships in the `numactl` package (2.0.18-1ubuntu0.24.04.1 on this +# build); its `--help` exits 1 with everything on stdout, no blank line +# anywhere and no `usage:` label. Maintainer-audited: "yeah this one needs +# full revision". Fixed; see docs/shapes.md S-165 (the root-description +# swallow), S-176 (the fabricated placeholder), S-177 (the positional +# recovery), and S-172 for the already-shipped `-rNUM` guard this fixture +# also exercises. [bless] provenance = "agent" @@ -70,7 +36,3 @@ must_contain_positionals = ["size"] [contract.must_value_name] "-f" = "FILE" - -[xfail] -broken = true -reason = "every one of the five raw lines is wrong: no positional is extracted, -f and -H are given fabricated value names read off their own descriptions, and all three flags' real descriptions plus the policy-choices line are swallowed into one root-level description blob instead of being attached to their own rows" diff --git a/corpus/ntfsfallocate/audit-seed2/expected.snap b/corpus/ntfsfallocate/audit-seed2/expected.snap index 6dcc3942..e054a0b8 100644 --- a/corpus/ntfsfallocate/audit-seed2/expected.snap +++ b/corpus/ntfsfallocate/audit-seed2/expected.snap @@ -1,5 +1,8 @@ name: ntfsfallocate -description: '/usr/bin/ntfsfallocate v2022.10.3 (libntfs-3g) Copyright (c) 2013-2014 Jean-Pierre Andre Allocate clusters to a specified attribute of a specified file. Developers'' email address: ntfs-3g-devel@lists.sf.net News, support and information: https://github.com/tuxera/ntfs-3g/' +description: |- + /usr/bin/ntfsfallocate v2022.10.3 (libntfs-3g) Copyright (c) 2013-2014 Jean-Pierre Andre Allocate clusters to a specified attribute of a specified file. + + Developers' email address: ntfs-3g-devel@lists.sf.net News, support and information: https://github.com/tuxera/ntfs-3g/ usage: - 'Usage: /usr/bin/ntfsfallocate [options] -l length device file [attr-type [attr-name]]' flags: diff --git a/corpus/wpa_cli/audit-seed/expected.snap b/corpus/wpa_cli/audit-seed/expected.snap index ef8cd5ae..602d3bbe 100644 --- a/corpus/wpa_cli/audit-seed/expected.snap +++ b/corpus/wpa_cli/audit-seed/expected.snap @@ -1,5 +1,4 @@ name: wpa_cli -description: 'commands:' usage: - wpa_cli [-p] [-i] [-hvBr] [-a] [-P] [-g] [-G] [-s] [command..] flags: diff --git a/docs/design.md b/docs/design.md index 8329540b..206195bf 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2983,6 +2983,27 @@ Re-measured with `xtask coverage --tools lldb-server,lldb-server-18` against the real installed binaries: `existence_fabrication_tools` reads 0 and `exist` reads 0 for both. Docs/shapes.md S-167. +**`memhog`'s whole screen was wrong or lost, S-177's own positional +recovery ships below the five-tool bar, S-176's own value-name rule +does not (2026-09-13).** Maintainer, after looking at mandible's screen +for the tool: "yeah this one needs full revision". `memhog --help` has +no blank line anywhere and no `usage:` label; every one of its five raw +lines was wrong or lost before this round (docs/shapes.md S-165, S-176, +S-177). S-176's own rule — a single-dash table row's real value comes +from the tool's own usage line, never a guess at the row's own text — +was first measured at 1 tool (`memhog` alone) and revised once a +full-`PATH` sweep on an earlier version of the rule found it threw away +genuine values on `host` (`-c`/`-N`/`-t`/`-W`/`-R`/`-m`/`-p`) that a +narrower usage-line reading now recovers correctly; the revised rule +reads tree-level at 5 tools (`memhog`, `host`, `kpartx`, `numastat`, +`savelog`), clearing the bar. S-177's own positional-flattening rule +stays at 1 tool, `memhog`, below the bar; a raw-text grep over +`audit/queue-captures/*/0.std*` reads 48 tools for its own row shape, an +explicit upper bound on the raw text rather than a tree-level count. +Both ship: `corpus/memhog/2.0.18` and `corpus/host/9.18.39` promote out +of (or never enter) `[xfail]`, every contract field they name passes, +and the corpus sweep and named controls stay byte-identical. + ### Deferred, with the reason each is not simply undone **Sub-case (b) of the `-h` fallback is unmeasured and must stay that way until diff --git a/docs/shapes.md b/docs/shapes.md index 50a04393..bcc165a4 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3485,34 +3485,43 @@ entry's `tools` field and nothing else. It does not get a new entry. use: X [:] [option] -a # default pointer acceleration (factor) -ac disable access control restrictions -- tools: Xvfb +- tools: Xvfb, memhog - handling: Fixed. `extract_description`'s own bound (`leading_prose_bound`) is a blank-line search with no notion of a - flags block at all; with no recognized `usage:` line and no blank line - anywhere in the document, it returns the whole document, so a - headingless table's rows land in the description as well as being - independently recovered by `scan_entries`. Narrowly bounded: only when - no blank line exists at all and no usage line was recognized does the + flags block at all; with no blank line anywhere in the document, it + returns the whole document, so a headingless table's rows land in the + description as well as being independently recovered by `scan_entries`. + Narrowly bounded: only when no blank line exists at all does the description scan now also stop at the first line `starts_attested_headingless_flag_block` (S-052's own recognizer) accepts as a real option row, so an ordinary document's already-correct, cheap bound pays nothing extra. Xvfb's own `use: X [:] [option]` line, an unusual `use:` label rather than `usage:`, is the - root cause `flags_block_start` never reaches on its own. This is the - same specimen S-164 fixes the group-duplication half of; landing this - fix first is what let S-163's `+word` and alternation rows reach column - 0 at all. + root cause `flags_block_start` never reaches on its own. `memhog` + (round 12) showed the bound was still too narrow: its first line IS a + recognized usage line (an unlabelled synopsis opening on its own name), + so the original gate (also requiring no usage line recognized) never + fired, and the flag rows after that one usage line still landed in the + description the same way. The `no usage line recognized` half of the + gate is dropped; `no blank line anywhere` is the only precondition left, + since that alone already keeps the bound cheap for an ordinary document. + This is the same specimen S-164 fixes the group-duplication half of; + landing this fix first is what let S-163's `+word` and alternation rows + reach column 0 at all. - fleet: `headingless-table-in-root-description` (`xtask/src/detector/headingless_table_in_root_description.rs`) is family `None`, so calibration reads NOT EVALUABLE. Self-checks hold - (4/4). One tool, below the five-tool bar, maintainer-named (carried - item 23). Full-`PATH` sweep-diff of 2269 tools against `origin/main` - 0b30c15, 2026-09-12: 0 flag/subcommand losses anywhere. The detector's - own word-boundary heuristic still reads 30 tools fleet-wide after this - fix — a broader symptom (a description repeating several of its own - tree's flag spellings) than the narrow structural cause this fix - closes (no blank line anywhere, no recognized usage line); left as a - future finding, not chased here. + (4/4). Two tools, below the five-tool bar, maintainer-named (Xvfb + carried item 23; memhog maintainer-audited in round 12, "yeah this one + needs full revision"). Full-`PATH` sweep-diff of 2269 + tools against `origin/main` 0b30c15, 2026-09-12: 0 flag/subcommand + losses anywhere. The detector's own word-boundary heuristic still reads + 30 tools fleet-wide after this fix — a broader symptom (a description + repeating several of its own tree's flag spellings) than the narrow + structural cause this fix closes (no blank line anywhere); left as a + future finding, not chased here. Round 12's own widening (dropping the + usage-line precondition) was not re-swept fleet-wide; reported, not + gated on a fresh number. ### S-166: header-declared three-column option table, env-variable column @@ -3916,3 +3925,114 @@ entry's `tools` field and nothing else. It does not get a new entry. tools have no fixture yet; rustc's own row is verified by a direct unit test (`grammar.rs`'s `a_nested_bracket_value_after_a_space_keeps_the_whole_outer_group`). + +### S-176: a table row with no placeholder takes its description's first word as a value + +- id: S-176 +- looks like: | + -f mmap is backed by FILE + -c specifies query class for non-IN data + -a is equivalent to -v -t ANY +- tools: memhog, host, kpartx, numastat, savelog +- handling: Fixed, inside a document with no `--long` row anywhere + ([`document_has_no_long_row`]) and at least two short-only `HelpText`- + sourced flags (`MIN_TABLE_ROWS`). A row with no genuine placeholder at + all still reads the first bare word of its own description as a value + (`-f mmap is backed by FILE` to `-f` valued `"mmap"`), and the real + description is lost outright once that first word is spent. The row's + own text is never trusted to say what the value is — the tool's own + usage line is, read by the exact grammar and bundle/alternation logic + [`extract_usage_flags`] already uses everywhere else + ([`usage_derived_value_for_short`]), covering all four usage spellings: + a glued uppercase run (`[-fFILE]` → `FILE`), a spaced placeholder + (`host`'s `[-c class]` → `class`), a glued optional group (`numastat`'s + `[-s[]]` → ``, Optional), and no placeholder at all, whether + alone (`[-r]`), in an alternation (`kpartx`'s `[-a|-d|-u|-l]`), or in a + bundle (`host`'s `[-aCdilrTvVw]`) — all three read boolean. A letter the + usage line never names at all (`host`'s own `-A`/`-s`/`-U`/`-4`/`-6`) + gets no change at all: no evidence, no guess. The row's own remainder + becomes the description either way, minus one leading occurrence of the + usage value when the row's own text happens to open with it too, in + whichever spelling that row uses for it — bare (`savelog`'s `-r rolldir + - use rolldir...`), bracketed (`lsof`'s `-F [f] select fields; -F? for + help`, whose Optional `f` would otherwise be printed twice) or angled + ([`strip_leading_value_spelling`]). A row packing a second + flag by a real two-plus-space column gap (`lsof`'s own multi-column + `-T`/`-U`/`-v` summary line) is refused rather than read as one row's + description; a single-spaced mention of another flag in ordinary prose + (`kpartx`'s `-l list partitions ... added by -a`) is not mistaken for + that shape. +- fleet: Tree-level: 5 tools (`memhog`, `host`, `kpartx`, `numastat`, + `savelog`), each verified by its own unit test in + `mandible-extract/src/help_text/sections/repair.rs` (`host` also by + `corpus/host/9.18.39`, promoted straight in — never `[xfail]`). Clears + the five-tool bar (AGENTS.md §3.1) on that count; not swept fleet-wide + with `xtask coverage` this round (build capacity was contended, and the + orchestrator owns the full-`PATH` sweep). A raw-text grep over + `audit/queue-captures/*/0.std*` for a single-dash-letter row followed by + one space and a lowercase bare word reads 62 tools — an explicit upper + bound on the raw shape, not a tree-level count; most of those 62 + (`ffmpeg`, `ffplay`, `ffprobe`, `python3.12`, `qemu-aarch64-static`, …) + carry a `--long` row elsewhere and never reach this repair's own gate. + Honest residual: `host`'s own `-A`, `-s`, `-U`, `-4`, `-6`, named in no + usage line at all, stay exactly as wrong as before — 5 rows on 1 tool, + recorded in `corpus/host/9.18.39/meta.toml` rather than claimed fixed. + Zero-loss checked against `git diff`'s own field-level list on every + corpus fixture this round: no `value_name` went from something real to + nothing anywhere except `memhog`'s own already-fenced S-172 case (S-177 + gained, `lsof`'s own `-F` gained a description with its value + untouched), and `Xvfb`/`mksquashfs`/`sqfstar` stayed byte-identical + (`pty_screenshot.py`, before/after diff empty). + +### S-177: a usage synopsis's lowercase operand tail, unlabelled and nested + +- id: S-177 +- looks like: | + memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]] +- tools: memhog +- handling: Fixed, for one narrow case: a single-physical-line *unlabelled* + synopsis (no `usage:` marker, no continuation line). `extract_positionals` + never reads a lowercase operand tail — its own ALL-CAPS/`<...>` loop + refuses `size[kmg]` and the nested `[policy [nodeset]]` outright, and the + existing tail-recovery fallbacks both decline it too: + `recover_primary_tail_operands`'s own bracket trim is not depth-aware and + refuses a nested group rather than misreading it; + `recover_trailing_multiword_operand` requires a sibling flag group + already showing an identical multi-word-value shape, which this usage + line has none of. `recover_lowercase_tail_positionals` + (`mandible-extract/src/help_text/sections/multiword.rs`) reads the + required lead word (`size`, its own glued bracket suffix `[kmg]` kept + visible in the verbatim `usage` text rather than folded into the name) + then flattens every further bracket group's words into an ordered list + of optional positionals — design §7 Tier B rule 17's own ruling for + `[A [B]]`, since position already carries the dependency. Gated on a new + `unlabelled_single_line` flag threaded through `extract_positionals` + (never inferred from `primary_lines` alone), so this fallback is never + reached for an ordinary labelled tool's own single-line form, whatever + convention that tool happens to use for its own operand names. The + primary-line bookkeeping that licenses positional recovery at all + (`primary_synopsis_lines`) was previously computed only for a labelled + usage block; a single-physical-line unlabelled synopsis now also + qualifies, since the existence oracle's own synopsis scanner + (`xtask::existence::synopsis_lines`) already attests such a line via the + same `looks_like_unlabeled_synopsis_line` the tier itself uses, so an + operand recovered from it is not reported as invented. `memhog`'s own + `Policies: preferred-many local interleave membind preferred default` + line, which the raw capture would otherwise simply drop once it stops + being swallowed into the root description (AGENTS.md §3.9), becomes the + `policy` positional's own `choices` via `attach_policies_line_choices` — + narrowly scoped to that literal label, not a generalized colon-list + recognizer (S-168's own general case was declined as materially larger + and riskier than this item's scope). +- fleet: Not measured with `xtask coverage` this round (build capacity was + contended; the orchestrator owns the full-`PATH` sweep). A raw-text grep + over `audit/queue-captures/*/0.std*` for an unlabelled single-line + synopsis whose tail carries a lowercase word followed by a nested + bracket pair reads 48 tools — an upper bound on the raw shape, not a + tree-level count; most carry a labelled `usage:` line or a multi-line + synopsis and so never reach `unlabelled_single_line`. Tree-level: one + tool, `memhog`, verified by the same + `memhog_flags_positionals_and_root_description_all_land_correctly` test. + Below the five-tool bar; shipped as a gated exception (docs/design.md + §16) alongside `corpus/memhog/2.0.18`, promoted out of `[xfail]`, + maintainer-audited ("yeah this one needs full revision"). diff --git a/mandible-extract/src/help_text/sections/mod.rs b/mandible-extract/src/help_text/sections/mod.rs index e7377b1b..168055e1 100644 --- a/mandible-extract/src/help_text/sections/mod.rs +++ b/mandible-extract/src/help_text/sections/mod.rs @@ -881,24 +881,44 @@ fn scan_usage_section( ); i += 1; } - // Scoped to a labelled block, never an unlabelled synopsis - // (`dbus-cleanup-sockets`, `lvreduce`): the existence oracle's own - // synopsis scanner has no unlabelled-synopsis support yet, so any - // operand recovered from one reports as invented today. See - // S-001. - let primary_lines = if labelled_usage_start.is_some() { - primary_synopsis_lines(&usage_entries, &line_entry_index, usage_lines.len()) - } else { - std::collections::HashSet::new() - }; + let positionals = finish_positional_recovery( + usage_lines, + labelled_usage_start, + &usage_entries, + &line_entry_index, + ); UsageScan { next_index: i, - positionals: extract_positionals(usage_lines, primary_lines), + positionals, entries: usage_entries, recovered_bare_root_stanza, } } +/// The tail of [`scan_usage_section`], split out to keep that function +/// under its own line ceiling (AGENTS.md §2). Scoped to a labelled block +/// for most unlabelled-synopsis shapes, with one exception: a +/// single-physical-line unlabelled synopsis (`memhog`) names its primary +/// form unambiguously, and the existence oracle already attests such a +/// line (S-001's own recognizer), so an operand recovered from it is not +/// reported as invented. See docs/shapes.md S-177. +fn finish_positional_recovery( + usage_lines: &[String], + labelled_usage_start: Option, + usage_entries: &[String], + line_entry_index: &[usize], +) -> Vec { + let unlabelled_single_line = labelled_usage_start.is_none() && usage_lines.len() == 1; + let primary_lines = if labelled_usage_start.is_some() { + primary_synopsis_lines(usage_entries, line_entry_index, usage_lines.len()) + } else if unlabelled_single_line { + std::collections::HashSet::from([0]) + } else { + std::collections::HashSet::new() + }; + extract_positionals_inner(usage_lines, primary_lines, unlabelled_single_line) +} + /// The leading prose before the usage block, as the node description. /// Paragraph-aware so a version or author banner can be dropped. /// See docs/shapes.md S-001. @@ -980,6 +1000,55 @@ fn extract_description( } } +/// Real prose that sits after every flags/choices row a headingless +/// document carries (`ar`'s own "supported targets: ..."/"Report bugs +/// to..." trailer, `ntfsfallocate`'s "Developers' email address: +/// .../News, support..."), recovered from the tail of the document +/// backward: a run of non-empty, column-0, non-flag-shaped lines ending +/// at the last line. Stops at the first blank line, indented line, or +/// flag-shaped/heading line, so it never reaches back into the table +/// itself — only genuine trailer text past every row `flag_start` and +/// `scan_entries` already own. See docs/shapes.md S-165. +fn trailing_prose_after_flags_block(lines: &[&str], flag_start: usize) -> Option { + let mut tail: Vec<&str> = Vec::new(); + let mut k = lines.len(); + while k > flag_start { + let idx = k - 1; + let l = lines[idx]; + if l.trim().is_empty() || leading_whitespace(l) != 0 { + break; + } + let t = l.trim_start(); + // A getopt-family "bad option" complaint (S-162), however it is + // decorated (Xvfb's own `(EE) Unrecognized option: --help`), is + // diagnostic noise about the probe's own `--help` argument, never + // real description prose. Discards the whole tail collected so + // far, not just this line: the diagnostic marks everything back + // to it as the same noisy preamble, not genuine trailer text. + // Substring, not `is_option_error_line`'s own line-opening test: + // that test's prefix rule refuses a decorated line like this one. + let lower = t.to_ascii_lowercase(); + let is_diagnostic = [ + "unknown option", + "invalid option", + "illegal option", + "unrecognized option", + ] + .iter() + .any(|kw| lower.contains(kw)); + if is_diagnostic { + return None; + } + if looks_like_flag_start(t) || is_section_heading_line(t) { + break; + } + tail.push(l); + k -= 1; + } + tail.reverse(); + non_empty_text(&tail.join("\n")).map(|t| t.as_str().to_string()) +} + /// Walk the document body after the description, emitting flags, /// subcommands, modifiers, positionals and environment variables as /// each recognized section shape is met. Returns the entry counts @@ -2102,6 +2171,16 @@ fn parse_body( // block just recovered, so it can never eat unrelated prose. See // docs/shapes.md S-127. i = apply_positional_description_block(&lines, i, &mut result.positionals); + // `memhog`'s own `Policies: preferred-many local interleave + // membind preferred default` enumerates what its own `policy` + // positional accepts — S-168's colon-introduced choice list + // shape, on the same line rather than an indented block beneath + // it. Narrowly scoped to that literal label, never generalized + // (S-168's own general case was declined as materially larger + // and riskier): recovers the choices this line would otherwise + // lose once it stops being swallowed whole into the root + // description (AGENTS.md §3.9). See docs/shapes.md S-177. + attach_policies_line_choices(&lines, &mut result.positionals); } // 2. Leading prose before the usage block (or before the first @@ -2114,22 +2193,32 @@ fn parse_body( // of milliseconds. let prose_bound = leading_prose_bound(&lines); let mut description_bound = i.max(prose_bound); - // A headingless table with no blank line ahead of it and no - // recognized usage line (Xvfb's `use: X [:] [option]`, not - // `usage:`) reaches `leading_prose_bound`'s whole-document fallback - // untouched, so its option rows land in the description as well as - // being independently recovered by `scan_entries` below — the same - // text rendered twice. Consulted only in that narrow case (no blank - // line anywhere, no usage line), so an ordinary document's already- - // correct, cheap bound pays nothing extra. See docs/shapes.md S-165. - if usage_start.is_none() && prose_bound == lines.len() { - if let Some(flag_start) = - (i..lines.len()).find(|&j| starts_attested_headingless_flag_block(&lines, j)) - { - description_bound = description_bound.min(flag_start); + // A headingless table with no blank line ahead of it reaches + // `leading_prose_bound`'s whole-document fallback untouched, so its + // option rows land in the description too, doubling text + // `scan_entries` below recovers anyway. Not gated on `usage_start` + // being absent (Xvfb has none; `memhog`'s one usage line covers only + // `i` lines, so the fallback still swallows what follows): the shared + // "no blank line anywhere" precondition is what keeps an ordinary + // document's bound cheap. The trailing run past the table (`ar`'s own + // "supported targets: ..." trailer) is separately recovered below and + // reattached, since bounding the whole scan would drop it. S-165. + let flag_start = (prose_bound == lines.len()) + .then(|| (i..lines.len()).find(|&j| starts_attested_headingless_flag_block(&lines, j))) + .flatten(); + if let Some(start) = flag_start { + description_bound = description_bound.min(start); + } + let mut description = extract_description(&lines, description_bound, usage_start, i); + if let Some(start) = flag_start { + if let Some(tail) = trailing_prose_after_flags_block(&lines, start) { + description = Some(match description { + Some(d) => format!("{d}\n\n{tail}"), + None => tail, + }); } } - if let Some(description) = extract_description(&lines, description_bound, usage_start, i) { + if let Some(description) = description { result.description = Some(description); } @@ -2142,6 +2231,15 @@ fn parse_body( }; let (total_entries, clean_entries) = scan_entries(&inp, tool_name, i, &mut result); + // A single-dash table row with no genuine placeholder (`memhog`'s + // `-f mmap is backed by FILE`) reads its own description's first word + // as a fabricated value and loses the description outright. Repaired + // before the usage-flag merge below, from the usage line's own glued + // spelling, so the merge's "let the described version win" rule sees + // an already-correct, already-described row rather than needing its + // own exception. See docs/shapes.md S-176. + recover_bare_word_first_description_word(&mut result.flags, &usage_lines, &lines); + // spec [M-15]: mine the usage synopsis for flag spellings too, not just // positionals — git's own flags documented only in its usage block // (378 of 1,895 `ok` tools fleet-wide had none read at all). Deferred @@ -2263,6 +2361,101 @@ fn compute_confidence(total_entries: usize, clean_entries: usize, had_usage: boo mod tests { use super::*; + /// `memhog --help`'s own bytes, byte-exact: an unlabelled synopsis + /// whose flag rows carry no column gap. The required `size` + /// positional and the flattened optional `policy`/`nodeset` pair + /// (design §7 Tier B rule 17), `-f`'s real value and description, + /// `-H`'s real description with no fabricated value, `-r`'s + /// already-correct `NUM` (S-172) now described too. `Policies:` is + /// not flag-shaped, so it still reaches the root description as well + /// as `policy`'s own `choices` — harmless, never a genuine loss. See + /// docs/shapes.md S-165, S-176, S-177, and corpus/memhog/2.0.18. + #[test] + fn memhog_flags_positionals_and_root_description_all_land_correctly() { + let raw = concat!( + "memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]\n", + "-f mmap is backed by FILE\n", + "-rNUM repeat memset NUM times\n", + "-H disable transparent hugepages\n", + "Policies: preferred-many local interleave membind preferred default\n", + ); + let parsed = parse_named(raw, "memhog"); + assert_eq!( + parsed.description.as_deref(), + Some("Policies: preferred-many local interleave membind preferred default"), + "only the non-flag-shaped Policies line reaches the root description" + ); + assert_eq!( + parsed + .positionals + .iter() + .map(|p| (p.primary_name().to_string(), p.required)) + .collect::>(), + vec![ + ("size".to_string(), true), + ("policy".to_string(), false), + ("nodeset".to_string(), false), + ] + ); + let policy = parsed + .positionals + .iter() + .find(|p| p.primary_name() == "policy") + .expect("policy positional"); + assert_eq!( + policy + .choices + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + vec![ + "preferred-many", + "local", + "interleave", + "membind", + "preferred", + "default" + ] + ); + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some('f')) + .expect("-f"); + assert_eq!(f.value_name.as_deref(), Some("FILE")); + assert_eq!( + f.description.as_ref().map(Text::as_str), + Some("mmap is backed by FILE") + ); + let r = parsed + .flags + .iter() + .find(|f| f.short() == Some('r')) + .expect("-r"); + assert_eq!(r.value_name.as_deref(), Some("NUM")); + assert_eq!(r.value_kind, ValueKind::Required); + assert_eq!( + r.description.as_ref().map(Text::as_str), + Some("repeat memset NUM times") + ); + let h = parsed + .flags + .iter() + .find(|f| f.short() == Some('H')) + .expect("-H"); + assert_eq!(h.value_name, None); + assert_eq!(h.value_kind, ValueKind::None); + assert_eq!( + h.description.as_ref().map(Text::as_str), + Some("disable transparent hugepages") + ); + assert_eq!(parsed.usage.len(), 1); + assert_eq!( + parsed.usage[0].as_str(), + "memhog [-fFILE] [-rNUM] [-H] size[kmg] [policy [nodeset]]" + ); + } + /// `lvcreate --help`'s own bytes, byte-exact: every invocation form /// reaches `usage` as its own alternative, and the tab-indented option /// rows beneath each form carry that form's own prose sentence as diff --git a/mandible-extract/src/help_text/sections/multiword.rs b/mandible-extract/src/help_text/sections/multiword.rs index 3d30e65a..f1194f9d 100644 --- a/mandible-extract/src/help_text/sections/multiword.rs +++ b/mandible-extract/src/help_text/sections/multiword.rs @@ -552,3 +552,153 @@ pub(super) fn recover_primary_tail_operands( }) .collect() } + +/// The primary synopsis's own trailing lowercase-convention operand run +/// (`memhog`'s own `size[kmg] [policy [nodeset]]`): a required lead word +/// (its own glued bracket suffix kept visible in the verbatim `usage` +/// text, never read as marking it optional), then bracket groups of +/// lowercase words flattened to an ordered list of optional positionals +/// (design §7 Tier B rule 17, `[A [B]]`). Neither existing tail recovery +/// reads this: [`recover_primary_tail_operands`]'s bracket trim is not +/// depth-aware; [`recover_trailing_multiword_operand`] needs a sibling +/// flag group in the identical shape. Scoped, like both, to the ALL-CAPS +/// loop above finding nothing, and further to a single-physical-line +/// *unlabelled* form only. See docs/shapes.md S-177. +pub(super) fn recover_lowercase_tail_positionals( + usage_lines: &[String], + primary_lines: &std::collections::HashSet, +) -> Vec { + let line_idx = match primary_lines.len() { + 1 => match primary_lines.iter().next() { + Some(&i) => i, + None => return Vec::new(), + }, + _ => return Vec::new(), + }; + let Some(line) = usage_lines.get(line_idx) else { + return Vec::new(); + }; + let before_desc = cut_before_description_gap(line); + let groups = group_synopsis_tokens(before_desc.trim()); + if groups.len() < 2 { + return Vec::new(); + } + // The program name, then every flag-shaped group (dash-led, whatever + // notation it carries); the first group after that which is neither + // opens the operand tail. Skips index 0 explicitly rather than + // filtering the result of `position()` — the program name itself + // also "doesn't start with `-`", so a plain `position()` would match + // it first and never look further. + let Some(tail_start) = groups + .iter() + .enumerate() + .skip(1) + .find(|(_, g)| !g.trim_matches(|c| c == '[' || c == ']').starts_with('-')) + .map(|(idx, _)| idx) + else { + return Vec::new(); + }; + let tail = &groups[tail_start..]; + // The lead operand must be required (no bracket around the whole + // group) and lowercase-word-shaped, its own glued bracket suffix + // aside. + let first = &tail[0]; + if first.starts_with('[') { + return Vec::new(); + } + let lead_base_end = first.find('[').unwrap_or(first.len()); + let lead_base = &first[..lead_base_end]; + if lead_base.is_empty() + || !lead_base + .chars() + .all(|c| c.is_ascii_lowercase() || c == '_' || c == '-') + { + return Vec::new(); + } + let mut lead = Entity::positional(lead_base.to_string(), Provenance::single(Source::HelpText)); + lead.required = true; + let mut out = vec![lead]; + for group in &tail[1..] { + let Some(words) = parse_nested_optional_words(group) else { + return out; // stop at the first group this shape doesn't read + }; + for w in words { + let mut p = Entity::positional(w, Provenance::single(Source::HelpText)); + p.required = false; + out.push(p); + } + } + out +} + +/// `memhog`'s own `Policies: preferred-many local interleave membind +/// preferred default` line, read as the `policy` positional's own +/// `choices`. Deliberately narrow — the literal label `Policies:`, not a +/// generalized colon-introduced-list recognizer (S-168's own general case +/// was declined as materially larger and riskier than this item's scope) +/// — so it can only ever help this one document. Never overwrites choices +/// a positional already carries. See docs/shapes.md S-177. +pub(super) fn attach_policies_line_choices(lines: &[&str], positionals: &mut [Entity]) { + let Some(policy) = positionals + .iter_mut() + .find(|p| p.primary_name() == "policy") + else { + return; + }; + if !policy.choices.is_empty() { + return; + } + let Some(rest) = lines + .iter() + .find_map(|l| l.trim().strip_prefix("Policies:")) + else { + return; + }; + let members: Vec = rest.split_whitespace().map(Choice::bare).collect(); + if !members.is_empty() { + policy.choices = members; + } +} + +/// Every lowercase word inside a usage-synopsis bracket group, in source +/// order, whatever its own nesting depth — the flattening design §7 Tier +/// B rule 17 requires for `[A [B]]`. `None` unless the group's own +/// brackets balance and every word is lowercase-word-shaped; a malformed +/// or non-lowercase group is left alone rather than guessed at. See +/// docs/shapes.md S-177. +fn parse_nested_optional_words(group: &str) -> Option> { + let mut words = Vec::new(); + let mut cur = String::new(); + let mut depth = 0i32; + for c in group.trim().chars() { + match c { + '[' => depth += 1, + ']' => { + if !cur.is_empty() { + words.push(std::mem::take(&mut cur)); + } + depth -= 1; + } + c if c.is_whitespace() => { + if !cur.is_empty() { + words.push(std::mem::take(&mut cur)); + } + } + c => cur.push(c), + } + } + if !cur.is_empty() { + words.push(cur); + } + if depth != 0 || words.is_empty() { + return None; + } + if !words.iter().all(|w| { + !w.is_empty() + && w.chars() + .all(|c| c.is_ascii_lowercase() || c == '_' || c == '-') + }) { + return None; + } + Some(words) +} diff --git a/mandible-extract/src/help_text/sections/repair.rs b/mandible-extract/src/help_text/sections/repair.rs index 95fbf182..b49358f6 100644 --- a/mandible-extract/src/help_text/sections/repair.rs +++ b/mandible-extract/src/help_text/sections/repair.rs @@ -394,6 +394,158 @@ pub(super) fn repair_usage_attested_single_dash_long(flags: &mut [Entity], usage } } +/// A single-dash table row with no genuine placeholder (`memhog`'s `-f +/// mmap is backed by FILE`) reads its description's first word as a +/// fabricated value; the row's own text is never trusted to say what a +/// value is. The tool's own usage line is +/// ([`usage_derived_value_for_short`], same grammar and bundle logic +/// [`extract_usage_flags`] already uses) or it is left alone — `host`'s +/// own `-A`/`-s`/`-U`/`-4`/`-6`, named in no usage line at all, are +/// exactly this refusal. Scoped, like S-157, to a document with no +/// `--long` row ([`document_has_no_long_row`]); the floor is a flag +/// count ([`MIN_TABLE_ROWS`]). See docs/shapes.md S-176. +pub(super) fn recover_bare_word_first_description_word( + flags: &mut [Entity], + usage_lines: &[String], + lines: &[&str], +) { + if !document_has_no_long_row(lines) { + return; + } + let short_only_help_text_flags = flags + .iter() + .filter(|f| { + f.provenance.sources.contains(&Source::HelpText) + && !f.provenance.sources.contains(&Source::HelpTextSynopsis) + && f.short().is_some() + && f.long().is_none() + }) + .count(); + if short_only_help_text_flags < MIN_TABLE_ROWS { + return; + } + for flag in flags.iter_mut() { + if !flag.provenance.sources.contains(&Source::HelpText) + || flag.provenance.sources.contains(&Source::HelpTextSynopsis) + { + continue; + } + if flag.description.is_some() || flag.long().is_some() { + continue; + } + let Some(short) = flag.short() else { continue }; + // No evidence, no change: a letter the usage line never spells + // at all keeps today's row-only guess rather than being assumed + // boolean. + let Some((usage_value, usage_kind)) = usage_derived_value_for_short(usage_lines, short) + else { + continue; + }; + let prefix = format!("-{short}"); + // Exactly one candidate row, never the first of several: `lsof` + // documents `-T` twice, once in a packed multi-column summary + // line and once in its own real row, and taking whichever comes + // first read the summary's neighbouring flags into `-T`'s own + // description. A row packing a second flag onto the same line + // (a further `-word` token past the first) is refused the same + // way, even when it is the only candidate. + let mut candidates = lines.iter().filter(|l| l.trim_start().starts_with(&prefix)); + let Some(row) = candidates.next() else { + continue; + }; + if candidates.next().is_some() { + continue; + } + let after_letter = &row.trim_start()[prefix.len()..]; + // A second flag token *packed onto the same line by a real column + // gap* (`lsof`'s `-T disable TCP/TPI info -U select Unix + // socket`, two-plus spaces before `-U`) is refused — that row + // documents more than one flag and taking the whole remainder + // would read a neighbour's row into this one's description. A + // single-spaced dash-word is ordinary prose mentioning another + // flag (`kpartx`'s own `-l list partitions ... added by -a`) and + // must not trip this guard. + if let Some(gap) = find_multi_space_gap(after_letter) { + if after_letter[gap..].trim_start().starts_with('-') { + continue; + } + } + // The row's own remainder is the description, whole, unless it + // happens to open with the usage-named value as its own leading + // token (`savelog`'s `-r rolldir - use rolldir...`: `rolldir` is + // both the value and the row's own first word) — then only that + // leading occurrence, and an immediately following ` - ` + // separator, are stripped off it + // ([`strip_leading_value_spelling`]). `host`'s `-c specifies + // query class for non-IN data` has no such overlap (`class`, the + // real value, never opens the row), so the whole sentence stays + // the description exactly as it reads. + let trimmed = after_letter.trim_start(); + let desc_source = match usage_value.as_deref() { + Some(v) if !v.is_empty() => strip_leading_value_spelling(trimmed, v).unwrap_or(trimmed), + _ => trimmed, + }; + let desc = desc_source.trim(); + if desc.is_empty() { + continue; + } + flag.value_name = usage_value; + flag.value_kind = usage_kind; + flag.description = non_empty_text(desc); + } +} + +/// The row's own leading occurrence of the usage-named value, stripped +/// off the row's remainder along with an immediately following ` - ` +/// separator, in whichever spelling the row itself uses for it: bare +/// (`savelog`'s `-r rolldir - use rolldir instead of .`), bracketed +/// (`lsof`'s `-F [f] select fields; -F? for help`, whose value is +/// already Optional and would otherwise be printed twice) or angled +/// (``). `None` when the row does not open with the value at all +/// (`host`'s `-c specifies query class for non-IN data`), and the whole +/// remainder is the description. The token must end at whitespace or at +/// the end of the row, so a longer word merely starting with the value's +/// own letters is never cut. See docs/shapes.md S-176. +fn strip_leading_value_spelling<'a>(trimmed: &'a str, value: &str) -> Option<&'a str> { + for spelling in [ + value.to_string(), + format!("[{value}]"), + format!("<{value}>"), + ] { + let Some(rest) = trimmed.strip_prefix(spelling.as_str()) else { + continue; + }; + if !rest.is_empty() && !rest.starts_with(char::is_whitespace) { + continue; + } + let rest = rest.trim_start(); + return Some(rest.strip_prefix("- ").unwrap_or(rest)); + } + None +} + +/// The value [`extract_usage_flags`] itself would assign `short`, read +/// straight off the usage line by the same grammar (a spaced placeholder, +/// `[-c class]`; a glued optional group, `[-s[]]`; a glued +/// uppercase run, `[-fFILE]`) and the same bundle/alternation logic +/// ([`push_usage_token`]) every other usage-derived flag already goes +/// through — never a second, narrower reading of the usage line built +/// just for this repair. `Some((None, ValueKind::None))` when the usage +/// line spells the letter with no placeholder at all, alone, in an +/// alternation, or in a bundle (`kpartx`'s `[-a|-d|-u|-l]`, `host`'s +/// `[-aCdilrTvVw]`). `None` when the usage line never names this letter, +/// so the caller leaves the row untouched rather than assuming boolean. +/// See docs/shapes.md S-176. +fn usage_derived_value_for_short( + usage_lines: &[String], + short: char, +) -> Option<(Option, ValueKind)> { + extract_usage_flags(usage_lines) + .into_iter() + .find(|f| f.short() == Some(short) && f.long().is_none()) + .map(|f| (f.value_name, f.value_kind)) +} + /// A dash-prefixed usage-line word that normally reads as the generic /// "any option" placeholder, or a swallowed-value split (`lshw`'s /// `-format`), is the tool's own literal flag when the document also @@ -499,20 +651,16 @@ const MIN_TABLE_ROWS: usize = 2; /// qualifies; only an already-unambiguous row may vouch for the /// document. See [`is_table_leading_token`]. fn single_dash_long_table(lines: &[&str]) -> bool { + if !document_has_no_long_row(lines) { + return false; + } let mut table_rows = 0usize; for (idx, line) in lines.iter().enumerate() { let trimmed = line.trim_start(); let Some(rest) = trimmed.strip_prefix('-') else { continue; }; - let Some(first) = rest.chars().next() else { - continue; - }; - if first == '-' { - // A leading `--word` row disqualifies the whole document. - if rest[1..].starts_with(|c: char| c.is_alphanumeric()) { - return false; - } + if rest.starts_with('-') { continue; } if is_unambiguous_long_name(rest) && row_is_table_shaped(lines, idx) { @@ -522,6 +670,27 @@ fn single_dash_long_table(lines: &[&str]) -> bool { table_rows >= MIN_TABLE_ROWS } +/// True when a document's own option rows never introduce a `--` row +/// (checked against each physical line's own leading token, so a stray +/// `--` in prose — mksquashfs's own "Can be used with dialog --gauge +/// etc." — is never mistaken for one). The GCC/Clang glued-value +/// convention always documents `--help`/`--version` somewhere, so this +/// is the one discriminator every single-dash-only repair in this file +/// rests on, whatever further per-row evidence a given repair needs on +/// top of it. See [`single_dash_long_table`]. +fn document_has_no_long_row(lines: &[&str]) -> bool { + for line in lines { + let trimmed = line.trim_start(); + let Some(rest) = trimmed.strip_prefix("--") else { + continue; + }; + if rest.starts_with(|c: char| c.is_alphanumeric()) { + return false; + } + } + true +} + /// True when `rest` (a row's own text, dash already stripped) opens with /// a name at least two characters long, every one of them a lowercase /// letter, digit, hyphen or underscore — a spelling no reasonable @@ -1006,6 +1175,213 @@ mod tests { ); } + /// `savelog --help`'s own bytes, byte-exact (minus the trailing rows + /// this test doesn't need): `-r rolldir`'s usage placeholder is + /// `rolldir` itself, a spaced usage-line value + /// ([`usage_derived_value_for_short`]'s own case), and `rolldir` also + /// happens to be the row's own leading word (`-r rolldir - use + /// rolldir...`) — the row's gap before its own ` - ` separator + /// collapsed to one space because `rolldir` is long, which used to + /// read it as a fabricated placeholder guess and throw the real + /// value away recovering the description. Both survive: value from + /// the usage line, description from the row's own remainder past + /// that leading occurrence. See docs/shapes.md S-176 and + /// audit/queue-captures/savelog/0.stdout. + #[test] + fn savelogs_own_dash_separator_keeps_a_narrow_gapped_value_and_recovers_its_description() { + let raw = concat!( + "Usage: savelog [-m mode] [-u user] [-g group] [-t] [-c cycle] [-p]\n", + " [-j] [-C] [-d] [-l] [-r rolldir] [-n] [-q] file ...\n", + "\t-m mode\t - chmod log files to mode\n", + "\t-u user\t - chown log files to user\n", + "\t-g group - chgrp log files to group\n", + "\t-c cycle - save cycle versions of the logfile (default: 7)\n", + "\t-r rolldir - use rolldir instead of . to roll files\n", + ); + let parsed = parse(raw); + let r = parsed + .flags + .iter() + .find(|f| f.short() == Some('r')) + .expect("-r must survive as its own short flag"); + assert_eq!( + r.value_name.as_deref(), + Some("rolldir"), + "the genuine value must never be thrown away" + ); + assert_eq!(r.value_kind, ValueKind::Required); + assert_eq!( + r.description.as_ref().map(Text::as_str), + Some("use rolldir instead of . to roll files") + ); + } + + /// `host --help`'s own bytes, byte-exact: a single-dash table with no + /// column gap at all, where every row used to read its description's + /// first word as a fabricated value. `-c`/`-N`/`-t`/`-W` each get + /// their real value (`class`/`ndots`/`type`/`time`) from `host`'s own + /// usage line's spaced placeholder; `-a` is a member of the usage + /// line's own bundle `[-aCdilrTvVw]` and is read boolean, its real + /// description recovered with no value fabricated. `-A`, named in no + /// usage line at all, is the deliberate residual this rule leaves + /// alone rather than guess at. See docs/shapes.md S-176 and + /// corpus/host/9.18.39. + #[test] + fn hosts_own_usage_line_supplies_every_valued_flags_real_name() { + let raw = concat!( + "Usage: host [-aCdilrTvVw] [-c class] [-N ndots] [-t type] [-W time]\n", + " [-R number] [-m flag] [-p port] hostname [server]\n", + " -a is equivalent to -v -t ANY\n", + " -A is like -a but omits RRSIG, NSEC, NSEC3\n", + " -c specifies query class for non-IN data\n", + " -N changes the number of dots allowed before root lookup is done\n", + " -t specifies the query type\n", + " -W specifies how long to wait for a reply\n", + ); + let parsed = parse_named(raw, "host"); + let short = |c: char| { + parsed + .flags + .iter() + .find(|f| f.short() == Some(c)) + .unwrap_or_else(|| panic!("no -{c} in {:?}", parsed.flags)) + }; + for (letter, value) in [('c', "class"), ('N', "ndots"), ('t', "type"), ('W', "time")] { + let f = short(letter); + assert_eq!( + f.value_name.as_deref(), + Some(value), + "-{letter} must keep the usage line's own value name" + ); + assert_eq!(f.value_kind, ValueKind::Required); + assert!( + f.description.is_some(), + "-{letter} must recover its real description too" + ); + } + let a = short('a'); + assert_eq!(a.value_name, None, "-a is boolean per the usage bundle"); + assert_eq!(a.value_kind, ValueKind::None); + assert_eq!( + a.description.as_ref().map(Text::as_str), + Some("is equivalent to -v -t ANY") + ); + // `-A` is named in no usage line at all (not in the bundle, not + // in its own bracket group): no evidence, no change. It keeps + // today's fabricated guess rather than being assumed boolean. + let big_a = short('A'); + assert_eq!(big_a.value_name.as_deref(), Some("is")); + assert!(big_a.description.is_none()); + } + + /// `kpartx --help`'s own bytes, byte-exact: every flag is boolean. + /// `-f`/`-g`/`-p`/`-r`/`-v`, each its own bracketed usage token + /// (`[-r]`), already lost their fabricated values to the row-only + /// repair; `-a`/`-d`/`-u`/`-l`, members of the usage line's own + /// alternation `[-a|-d|-u|-l]`, did not — the alternation shape is + /// this rule's own addition. `-l`'s own row, "list partitions ... + /// added by -a", is prose merely mentioning another flag and must + /// not trip the packed-multi-column-row guard the way a real + /// two-plus-space gap would. See docs/shapes.md S-176 and + /// corpus/kpartx (unfixtured; captured live, `kpartx` prints its own + /// usage with no arguments). + #[test] + fn kpartxs_alternation_members_are_read_boolean_not_valued() { + let raw = concat!( + "Usage:\n", + " kpartx [-a|-d|-u|-l] [-r] [-p] [-f] [-g] [-s|-n] [-v] wholedisk\n", + "\t-a add partition devmappings\n", + "\t-r devmappings will be readonly\n", + "\t-d del partition devmappings\n", + "\t-u update partition devmappings\n", + "\t-l list partitions devmappings that would be added by -a\n", + "\t-v verbose\n", + ); + let parsed = parse_named(raw, "kpartx"); + for letter in ['a', 'd', 'u', 'l', 'r', 'v'] { + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some(letter)) + .unwrap_or_else(|| panic!("no -{letter} in {:?}", parsed.flags)); + assert_eq!(f.value_name, None, "-{letter} must be read boolean"); + assert_eq!(f.value_kind, ValueKind::None); + assert!( + f.description.is_some(), + "-{letter} must still recover its real description" + ); + } + } + + /// `lsof -h`'s own bytes, byte-exact (the rows this test needs): + /// `-F`'s usage placeholder is the optional `[f]`, and the row's own + /// remainder opens with that same `[f]` — the description must be + /// the sentence alone, never the placeholder printed a second time + /// beside the value the flag already carries. The packed + /// multi-column `-T`/`-U`/`-v` summary line is here too: `-T` has + /// two candidate rows and must keep the real value `fqs` from its + /// own row, untouched by this repair. See docs/shapes.md S-176 and + /// corpus/lsof/4.95.0. + #[test] + fn lsofs_bracketed_optional_value_is_not_printed_twice_in_its_description() { + let raw = concat!( + "lsof 4.95.0\n", + " usage: [-?abhKlnNoOPRtUvVX] [+|-c c] [+|-d s] [+D D] [+|-E] [+|-e s] [+|-f[gG]]\n", + " [-F [f]] [-g [s]] [-i [i]] [+|-L [l]] [+m [m]] [+|-M] [-o [o]] [-p s]\n", + " [+|-r [t]] [-s [p:s]] [-S [t]] [-T [t]] [-u s] [+|-w] [-x [fl]] [--] [names]\n", + " -T disable TCP/TPI info -U select Unix socket -v list version info\n", + " -F [f] select fields; -F? for help \n", + " -T fqs TCP/TPI Fl,Q,St (s) info\n", + " -g [s] exclude(^)|select and print process group IDs\n", + ); + let parsed = parse_named(raw, "lsof"); + let f = parsed + .flags + .iter() + .find(|f| f.short() == Some('F')) + .unwrap_or_else(|| panic!("no -F in {:?}", parsed.flags)); + assert_eq!(f.value_name.as_deref(), Some("f")); + assert_eq!(f.value_kind, ValueKind::Optional); + assert_eq!( + f.description.as_ref().map(Text::as_str), + Some("select fields; -F? for help"), + "-F's own bracketed placeholder must not open its description" + ); + } + + /// `numastat --help`'s own bytes, byte-exact: `-s[]` is a + /// glued optional group on the usage line, the third of the rule's + /// four usage spellings (a bare uppercase run, a spaced placeholder, + /// a glued optional group, no placeholder at all). `-p |` + /// is a spaced placeholder carrying its own alternation. See + /// docs/shapes.md S-176. + #[test] + fn numastats_glued_optional_group_keeps_its_own_bracket_value() { + let raw = concat!( + "Usage: numastat [-c] [-p |] [-s[]] [-v]\n", + "-c to minimize column widths\n", + "-p | to show process info\n", + "-s[] to sort data by total column or \n", + "-v to make some reports more verbose\n", + ); + let parsed = parse_named(raw, "numastat"); + let s = parsed + .flags + .iter() + .find(|f| f.short() == Some('s')) + .expect("no -s in numastat's flags"); + assert_eq!(s.value_kind, ValueKind::Optional); + assert!(s.value_name.as_deref().is_some_and(|v| v.contains("node"))); + assert!(s.description.is_some()); + let p = parsed + .flags + .iter() + .find(|f| f.short() == Some('p')) + .expect("no -p in numastat's flags"); + assert_eq!(p.value_kind, ValueKind::Required); + assert!(p.description.is_some()); + } + /// A spaced value is indistinguishable from a glued one once stored; /// the raw text is what decides. #[test] diff --git a/mandible-extract/src/help_text/sections/usage.rs b/mandible-extract/src/help_text/sections/usage.rs index fe628356..7f8f4f2f 100644 --- a/mandible-extract/src/help_text/sections/usage.rs +++ b/mandible-extract/src/help_text/sections/usage.rs @@ -239,6 +239,22 @@ pub(super) fn token_marks_repetition(token: &str) -> bool { pub(super) fn extract_positionals( usage_lines: &[String], primary_lines: std::collections::HashSet, +) -> Vec { + extract_positionals_inner(usage_lines, primary_lines, false) +} + +/// `unlabelled_single_line`: true only for the one shape `extract_positionals` +/// itself never sees anywhere else — a single-physical-line unlabelled +/// synopsis (`memhog`). Kept as its own parameter, never inferred from +/// `primary_lines` alone, so [`recover_lowercase_tail_positionals`] stays +/// scoped to exactly the case measured (docs/shapes.md S-177) and is +/// never reached for an ordinary labelled tool's own single-line form, +/// whatever convention that tool happens to use for its own operand +/// names. +pub(super) fn extract_positionals_inner( + usage_lines: &[String], + primary_lines: std::collections::HashSet, + unlabelled_single_line: bool, ) -> Vec { let mut seen = std::collections::HashSet::new(); let mut out = Vec::new(); @@ -406,6 +422,12 @@ pub(super) fn extract_positionals( &primary_lines, )); } + if out.is_empty() && unlabelled_single_line { + out.extend(recover_lowercase_tail_positionals( + usage_lines, + &primary_lines, + )); + } out }