feat(go): refuse undeclared environment variables in generated parsers

V8.md

@@ -88,3 +88,16 @@ derive keeps explicit names, and for a short-only flag it holds the short letter
 and the redeclaration check read. A saving here would drop `name` only where it equals
 `longs[0]` and keep a fallback for every other flag. `ArgMeta` (456 bytes) embeds its rarely used
 fields inline instead of behind a shared extra record, the way `FlagMeta` uses `FlagExtra`.
+
+## Go generation that can fail
+
+`usage::go::generate` returns a `String`, so it has no way to refuse a spec it cannot generate
+correctly. A spec with both `unknown_env` and a `mount` is the first such case: the Go generator
+does not emit mounted commands, so it cannot know their variables, and the generated parser
+would refuse variables usage-lib accepts. `generate_with_types`, which the CLI calls, already
+returns an error for it.
+
+- **What changes:** `generate` returns `Result<String, String>` and refuses what
+  `generate_with_types` refuses.
+- **Why it breaks:** the return type of a public function changes.
+- **What adopters do:** handle the error, or call `generate_with_types` with an empty map today.

docs/go/parser.md

@@ -69,6 +69,32 @@ parse with `CodeHelp` and `Error.Cmd` set to the command asked about. `Error.Lon
 Counting flags need nothing from the parser — each occurrence is its own event, and the caller
 tallies (generated code does `field++`).
 
+## Unknown environment variables
+
+A spec with a root-level `unknown_env` makes generated `Parse` refuse environment variables
+under that prefix that nothing reads:
+
+```kdl
+unknown_env prefix="MOQ_" {
+  allow "MOQ_VAAPI_DEVICE"
+}
+```
+
+A variable is known when any flag or argument anywhere in the command tree, clauses included,
+names it as `env`, `env_fallback` or `deprecated_env`, when a `config` property reads it, or
+when `allow` lists it. The generator works that list out once and emits it as
+`UnknownEnvKnown`, beside `UnknownEnvPrefix`. After help and version have been answered, and
+before any requirement is checked, `Parse` fails with `CodeUnknownEnv`: `Error.Name` is the
+prefix and `Error.Names` every offending variable, sorted. On Windows the comparison is
+case-insensitive.
+
+The check is `argv.RefuseUnknownEnv(prefix, known, argv.EnvNames())`, and `argv.UnknownEnv`
+answers the same question over any list of names, which is how to test it without touching the
+process environment.
+
+A spec that has both `unknown_env` and a `mount` is refused: the generator does not emit mounted
+commands, so it cannot know the variables they declare.
+
 ## Limits
 
 - `argv.MaxDepth` is 16; deeper command trees fail with `CodeTooDeep`. (mise is 4 deep.)

go/argv/argv.go

@@ -27,6 +27,8 @@
 // [conformance corpus]: https://github.com/jdx/usage/tree/main/corpus
 package argv
 
+import "strings"
+
 // MaxDepth is how deep a command tree the parser will descend.
 //
 // The ancestor chain lives in a fixed-size array so that a parse allocates
@@ -422,6 +424,10 @@ const (
 	// CodeDuplicateFlag means a strict command received a single-valued flag
 	// more than once.
 	CodeDuplicateFlag
+	// CodeUnknownEnv means the environment holds variables under the spec's
+	// `unknown_env` prefix that nothing declares. Name is the prefix, Names the
+	// variables. Raised by [RefuseUnknownEnv], after help and version.
+	CodeUnknownEnv
 )
 
 var codeNames = [...]string{
@@ -441,6 +447,7 @@ var codeNames = [...]string{
 	CodeConflictingFlags:      "conflicting_flags",
 	CodeInvalidValue:          "invalid_value",
 	CodeDuplicateFlag:         "duplicate_flag",
+	CodeUnknownEnv:            "unknown_env",
 }
 
 // String gives the code the corpus spells it with.
@@ -509,6 +516,8 @@ type Error struct {
 	// Both are carried because either alone reads as a puzzle: which flag is
 	// unwelcome depends on what else was given.
 	Other string
+	// Names is every offending variable for CodeUnknownEnv, sorted.
+	Names []string
 }
 
 // Error is the message Go's own error interface asks for.
@@ -556,6 +565,17 @@ func (e *Error) Error() string {
 			message += ": " + e.Reason
 		}
 		return message
+	case CodeUnknownEnv:
+		// Through `safe` because the environment is as much outside input as argv.
+		var message strings.Builder
+		message.WriteString("unknown environment " + pluralWord(len(e.Names), "variable") + " under " + e.Name + ": ")
+		for i, name := range e.Names {
+			if i > 0 {
+				message.WriteString(", ")
+			}
+			message.WriteString(safe(name))
+		}
+		return message.String()
 	}
 	return "parse error"
 }

go/argv/env.go

@@ -0,0 +1,128 @@
+package argv
+
+import (
+	"os"
+	"runtime"
+	"slices"
+	"strings"
+	"unicode/utf8"
+)
+
+// UnknownEnv is the variables in names that start with prefix and are not in
+// known, sorted and each once.
+//
+// A spec's `unknown_env` asks for this: a misspelled or stale variable under the
+// CLI's own prefix is read by nobody, and the program runs on its defaults. known
+// is every name the spec declares anywhere in its command tree, and its allow
+// list; the generator collects it, since the whole tree counts and not only the
+// command that ran — a variable meant for `serve` is not a typo when the user runs
+// `publish`.
+//
+// names is passed in rather than read from the process for the reason [Fill]
+// takes its lookup: a test of a parse should not be a test of the machine it runs
+// on. [EnvNames] reads the process. On Windows, where variable names are
+// case-insensitive, so is the comparison, and a variable set in two cases is
+// reported once.
+func UnknownEnv(prefix string, known, names []string) []string {
+	return unknownEnv(prefix, known, names, runtime.GOOS == "windows")
+}
+
+// unknownEnv is [UnknownEnv] with the platform as a parameter, so the Windows
+// rules can be tested anywhere.
+func unknownEnv(prefix string, known, names []string, fold bool) []string {
+	key := func(name string) string {
+		if fold {
+			return envKey(name)
+		}
+		return name
+	}
+	// An empty prefix owns nothing: the spec refuses one, and a hand-built call
+	// that passes one must not make PATH an error.
+	if prefix == "" {
+		return nil
+	}
+	var unknown []string
+	for _, name := range names {
+		// The prefix is matched against the valid UTF-8 head of the name, as
+		// usage-lib does: a prefix ending in U+FFFD must not claim a name whose
+		// invalid byte only reads as one.
+		if !strings.HasPrefix(key(validHead(name)), key(prefix)) {
+			continue
+		}
+		// A name that is not UTF-8 is never a declared one.
+		if utf8.ValidString(name) && slices.ContainsFunc(known, func(k string) bool { return key(k) == key(name) }) {
+			continue
+		}
+		unknown = append(unknown, name)
+	}
+	// On Windows EX_TYPO and ex_typo are one variable, and listing it twice would
+	// read as two mistakes. Names that are not UTF-8 are the same only byte for
+	// byte, since folding would turn every invalid byte into U+FFFD.
+	same := func(a, b string) bool {
+		if utf8.ValidString(a) && utf8.ValidString(b) {
+			return key(a) == key(b)
+		}
+		return a == b
+	}
+	// Sorted by the folded spelling and then the raw bytes, so the names same
+	// calls equal are always adjacent.
+	slices.SortFunc(unknown, func(a, b string) int {
+		if c := strings.Compare(key(strings.ToValidUTF8(a, "�")), key(strings.ToValidUTF8(b, "�"))); c != 0 {
+			return c
+		}
+		return strings.Compare(a, b)
+	})
+	return slices.CompactFunc(unknown, same)
+}
+
+// validHead is name up to its first byte that is not valid UTF-8.
+func validHead(name string) string {
+	for i, r := range name {
+		if r == utf8.RuneError {
+			if _, size := utf8.DecodeRuneInString(name[i:]); size == 1 {
+				return name[:i]
+			}
+		}
+	}
+	return name
+}
+
+// RefuseUnknownEnv is [UnknownEnv] as the error a parse returns, with every
+// offending variable named at once: a deployment with two typos should not need
+// two restarts to find them. It returns nil when there are none.
+func RefuseUnknownEnv(prefix string, known, names []string) error {
+	unknown := UnknownEnv(prefix, known, names)
+	if len(unknown) == 0 {
+		return nil
+	}
+	return &Error{Code: CodeUnknownEnv, Name: prefix, Names: unknown}
+}
+
+// EnvNames is the names of the process environment, for callers that want it.
+func EnvNames() []string {
+	environ := os.Environ()
+	names := make([]string, 0, len(environ))
+	for _, entry := range environ {
+		// Windows keeps per-drive working directories as `=C:=C:\dir`, so the
+		// separator is the first `=` after the name's first byte.
+		if i := strings.IndexByte(entry[min(1, len(entry)):], '='); i >= 0 {
+			entry = entry[:i+min(1, len(entry))]
+		}
+		names = append(names, entry)
+	}
+	return names
+}
+
+// envKey is a Windows variable name as compared: uppercased with Unicode rules,
+// as usage-lib does, because Windows folds ä and Ä together too. Not
+// strings.EqualFold, which also folds the Kelvin sign to K where usage-lib and
+// this key do not, and which no sort order can agree with.
+func envKey(name string) string { return strings.ToUpper(name) }
+
+// pluralWord is the noun alone, without [plural]'s count.
+func pluralWord(n int, noun string) string {
+	if n == 1 {
+		return noun
+	}
+	return noun + "s"
+}

go/argv/env_test.go

@@ -0,0 +1,64 @@
+package argv
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+)
+
+func TestUnknownEnv(t *testing.T) {
+	known := []string{"EX_PORT", "EX_LIBRARY"}
+	names := []string{"EX_TSL", "HOME", "EX_PORT", "EX_", "EX", "EX_PROT", "EX_TSL", "EXTRA"}
+	got := UnknownEnv("EX_", known, names)
+	if want := []string{"EX_", "EX_PROT", "EX_TSL"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %q, want %q", got, want)
+	}
+	if err := RefuseUnknownEnv("EX_", known, []string{"EX_PORT"}); err != nil {
+		t.Fatalf("nothing unknown should be a nil error, got %#v", err)
+	}
+	// Windows names are case-insensitive; elsewhere a different case is a different variable.
+	got = UnknownEnv("EX_", known, []string{"EX_port", "Ex_Port", "ex_other"})
+	want := []string{"EX_port"}
+	if runtime.GOOS == "windows" {
+		want = []string{"ex_other"}
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("case: got %q, want %q", got, want)
+	}
+	// The Windows rules, wherever this runs: one variable set in two cases is
+	// reported once, and the Kelvin sign is not K, so EX_K is its own name
+	// and stays apart from EX_K even with EX_Z sorted between them.
+	got = unknownEnv("EX_", nil, []string{"EX_K", "EX_Z", "EX_K", "ex_k", "EX_ä", "EX_Ä"}, true)
+	if want := []string{"EX_K", "EX_Z", "EX_Ä", "EX_K"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("windows: got %q, want %q", got, want)
+	}
+	// ß has no one-letter uppercase, so usage-lib keeps it as it is; so must this.
+	if got := envKey("EX_ß"); got != "EX_ß" {
+		t.Fatalf("ß folded to %q", got)
+	}
+	// Names that are not UTF-8: the prefix is matched on the valid head, so a
+	// prefix ending in U+FFFD does not claim an invalid byte; such a name is never
+	// a declared one; and two of them are one only when their bytes are.
+	for _, fold := range []bool{false, true} {
+		if got := unknownEnv("EX_�", nil, []string{"EX_\xff"}, fold); got != nil {
+			t.Fatalf("fold=%v: U+FFFD prefix claimed an invalid byte: %q", fold, got)
+		}
+		got := unknownEnv("EX_", []string{"EX_�"}, []string{"EX_\xfe", "EX_\xff", "EX_\xff"}, fold)
+		if want := []string{"EX_\xfe", "EX_\xff"}; !reflect.DeepEqual(got, want) {
+			t.Fatalf("fold=%v: non-UTF-8 names: got %q, want %q", fold, got, want)
+		}
+	}
+	if got := UnknownEnv("", known, []string{"PATH"}); got != nil {
+		t.Fatalf("an empty prefix owns nothing, got %q", got)
+	}
+}
+
+func TestUnknownEnvRendering(t *testing.T) {
+	err := &Error{Code: CodeUnknownEnv, Name: "EX_", Names: []string{"EX_PROT"}}
+	if got, want := err.Error(), "unknown environment variable under EX_: EX_PROT"; got != want {
+		t.Fatalf("Error(): %q", got)
+	}
+	if got, want := Render(err, nil, nil, nil), "error: unknown environment variable under `EX_`: `EX_PROT`\n\nFor more information, try `--help`.\n"; got != want {
+		t.Fatalf("Render: %q", got)
+	}
+}

go/argv/render.go

@@ -123,6 +123,13 @@ func explain(err *Error, help HelpTable) string {
 			typedAs(err.OtherSpelling, other) + "` cannot be given together"
 	case CodeDuplicateFlag:
 		return "`" + typedAs(err.Spelling, err.Name) + "` cannot be used multiple times"
+	case CodeUnknownEnv:
+		names := make([]string, len(err.Names))
+		for i, name := range err.Names {
+			names[i] = "`" + safe(name) + "`"
+		}
+		return "unknown environment " + pluralWord(len(names), "variable") + " under `" + err.Name + "`: " +
+			strings.Join(names, ", ")
 	}
 	return "the command line could not be parsed"
 }

go/conformance/generated_unknown_env_test.go

@@ -0,0 +1,68 @@
+package conformance
+
+import "testing"
+
+// TestGeneratedUnknownEnv runs a generated Parse against the real process
+// environment, which is what `unknown_env` judges.
+func TestGeneratedUnknownEnv(t *testing.T) {
+	runGeneratedFixture(t, `name "example"
+bin "example"
+version "1.0"
+unknown_env prefix="EX_" {
+  allow "EX_LIBRARY"
+}
+flag "--port <port>" env="EX_PORT"
+cmd "serve" {
+  flag "--tls" deprecated_env="EX_SSL"
+}
+cmd "publish" {
+  flag "--name <name>" required=#true env="EX_NAME"
+}
+config {
+  prop "jobs" env="EX_JOBS"
+}
+`, `package generated
+import (
+  "errors"
+  "os"
+  "reflect"
+  "strings"
+  "testing"
+  "github.com/jdx/usage/go/argv"
+)
+func clearEx(t *testing.T) {
+  for _, entry := range os.Environ() {
+    // Upper-cased, since on Windows an inherited ex_typo is under EX_ too.
+    if name, _, _ := strings.Cut(entry, "="); strings.HasPrefix(strings.ToUpper(name), "EX_") {
+      t.Setenv(name, "")
+      os.Unsetenv(name)
+    }
+  }
+}
+func TestUnknownEnv(t *testing.T) {
+  clearEx(t)
+  // Declared by another command, deprecated, read by config, allowed, or outside the prefix.
+  for _, name := range []string{"EX_PORT", "EX_SSL", "EX_JOBS", "EX_LIBRARY", "EXTRA"} { t.Setenv(name, "1") }
+  t.Setenv("EX_NAME", "n")
+  if _, err := Parse([]string{"publish"}); err != nil { t.Fatalf("declared names: %v", err) }
+
+  t.Setenv("EX_TSL", "1")
+  t.Setenv("EX_PROT", "1")
+  _, err := Parse([]string{"serve"}); var req *argv.Error
+  if !errors.As(err, &req) || req.Code != argv.CodeUnknownEnv { t.Fatalf("typos: %v", err) }
+  if req.Name != "EX_" || !reflect.DeepEqual(req.Names, []string{"EX_PROT", "EX_TSL"}) { t.Fatalf("every offender, sorted: %#v", req) }
+  if got, want := err.Error(), "unknown environment variables under EX_: EX_PROT, EX_TSL"; got != want { t.Fatalf("message: %q", got) }
+
+  // Reported ahead of the requirement it may be the cause of.
+  os.Unsetenv("EX_NAME")
+  _, err = Parse([]string{"publish"})
+  if !errors.As(err, &req) || req.Code != argv.CodeUnknownEnv { t.Fatalf("before requirements: %v", err) }
+
+  // Help and version still answer.
+  for _, tc := range []struct{ args []string; code argv.Code }{{[]string{"--help"}, argv.CodeHelp}, {[]string{"help", "serve"}, argv.CodeHelp}, {[]string{"--version"}, argv.CodeVersion}} {
+    _, err = Parse(tc.args)
+    if !errors.As(err, &req) || req.Code != tc.code { t.Fatalf("%v: %v", tc.args, err) }
+  }
+}
+`, "unknown-env")
+}