diff --git a/.github/ci/atlas_regime_gate.ps1 b/.github/ci/atlas_regime_gate.ps1 index db5644298..2cdc2c8b2 100644 --- a/.github/ci/atlas_regime_gate.ps1 +++ b/.github/ci/atlas_regime_gate.ps1 @@ -126,10 +126,14 @@ function Invoke-Gate([hashtable]$mutate) { # --- 3. the atlas ceiling is EXPLICIT ------------------------------------ # ImGui defaults TexMax to 8192. Inheriting it raises the worst DX12 upload - # behind an unbounded fence wait from 16.8 MB to 268 MB. + # behind an unbounded fence wait from 16.8 MB to 268 MB. Two ceilings are + # pinned: 2048, and 4096 under a script face, whose peer text accumulates + # (measured by the l10n drill's atlas arm); both are named + # constants and both sides of the texture take them. $f = $text['src\votv-coop\src\ui\fonts.cpp'] - if ($f -notmatch 'TexMaxWidth\s*=\s*2048' -or $f -notmatch 'TexMaxHeight\s*=\s*2048') { - $fails += "fonts.cpp does not pin TexMaxWidth/TexMaxHeight to 2048 (the 8192 default would be inherited)" + if ($f -notmatch 'kAtlasMax\s*=\s*2048' -or $f -notmatch 'kAtlasMaxScript\s*=\s*4096' -or + $f -notmatch 'TexMaxWidth\s*=\s*io\.Fonts->TexMaxHeight\s*=\s*ActiveScript\(\)\s*\?\s*kAtlasMaxScript\s*:\s*kAtlasMax') { + $fails += "fonts.cpp does not pin TexMaxWidth/TexMaxHeight to 2048 and 4096 (the 8192 default would be inherited)" } if ($f -notmatch 'GlyphExcludeRanges') { $fails += "fonts.cpp no longer sets GlyphExcludeRanges -- every codepoint in every face would bake" @@ -167,7 +171,15 @@ if ($Drill) { want = 'no longer asserts the regime' } @{ name = 'TexMax inherited' mut = @{ 'src\votv-coop\src\ui\fonts.cpp' = { - param($t) $t -replace 'TexMaxWidth\s*=\s*2048', 'TexMaxWidth = 8192' } } + param($t) $t -replace 'kAtlasMax\s*=\s*2048', 'kAtlasMax = 8192' } } + want = 'does not pin TexMaxWidth' } + @{ name = 'script ceiling inherited' + mut = @{ 'src\votv-coop\src\ui\fonts.cpp' = { + param($t) $t -replace 'kAtlasMaxScript\s*=\s*4096', 'kAtlasMaxScript = 8192' } } + want = 'does not pin TexMaxWidth' } + @{ name = 'ceiling not chosen by the script' + mut = @{ 'src\votv-coop\src\ui\fonts.cpp' = { + param($t) $t -replace 'ActiveScript\(\)\s*\?\s*kAtlasMaxScript', 'false ? kAtlasMaxScript' } } want = 'does not pin TexMaxWidth' } @{ name = 'exclude list a no-op' mut = @{ 'src\votv-coop\include\coop\text\exclude_ranges.inc' = { diff --git a/.github/ci/data_paths_gate.py b/.github/ci/data_paths_gate.py new file mode 100644 index 000000000..e3db7bf65 --- /dev/null +++ b/.github/ci/data_paths_gate.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""data_paths_gate -- the mod's own data is built from its own folder, never from the exe's. + +Everything the mod keeps for itself (the ini, the server profile, the player profiles, the seen list, +the thanks cache, the identity key) lives in `ue_wrap::paths::ModDataDir()`, so a copied game folder +carries none of it and a manager's reinstall starts clean. `ue_wrap::paths::ExeDir()` is the game +exe's folder: it stays the home of the logs, the reports, the dumps and what a player authors (a +translator's `.po` files). A data builder written against `ExeDir()` again would put a file back +beside the exe, where a copy of the game folder carries it and a reinstall misses it. This gate holds, +over `src/votv-coop/src` (comments and string literals set aside): + + 1. a call of `ExeDir(` stands only in the files of EXE_DIR_FILES, each of which builds a file that + belongs beside the exe (a log, a report, a diagnostic, a rig signal, a translator's input, the + game's own files read); + 2. a `GetModuleFileName`, `W` or `A`, whose module argument is null (`nullptr`, `NULL`, `0`, or a + `GetModuleHandle` of null) -- the exe's PATH asked of Windows directly, the same folder by + another road -- stands only in the files of EXE_PATH_FILES, all four of which read only. + A `GetModuleHandle(nullptr)` used for the module's base is not a path and is not matched. + +The lists are keyed by file, not by line: a line moves with every edit. A file added to a list is a +reviewed decision. + + python .github/ci/data_paths_gate.py # PASS/FAIL over the tree + python .github/ci/data_paths_gate.py --tree # the same over /src/votv-coop/src + python .github/ci/data_paths_gate.py --drill # the gate must fail each planted fault +""" +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] + +# What each file builds from ExeDir(): the reason it stays beside the exe. +EXE_DIR_FILES = { + "ue_wrap/core/paths.cpp": "ExeDir itself, and ExistsBesideExe", + "bootstrap/boot.cpp": "multivoid-loaded.txt, the install marker", + "ue_wrap/core/log.cpp": "multivoid.log and multivoid.prev.log", + "harness/compat_report.cpp": "the compat report", + "coop/bug_report/report_files.cpp": "the bundle's entry for the compat report", + "coop/bug_report/report_bundle.cpp": "multivoid_reports", + "harness/fingerprint_capture.cpp": "the fingerprint capture, a dev diagnostic", + "harness/screenshot.cpp": "coop-screenshots, a dev diagnostic", + "coop/dev/dependency_dump.cpp": "the dev dump", + "coop/dev/rehost_rejoin.cpp": "the rejoin trigger, a rig signal", + "l10n/l10n.cpp": "multivoid_locale, a translator's own files, read only", + "coop/player/skin_registry.cpp": "Content/Paks/LogicMods, the game's pak tree, read only", + "harness/mod_environment.cpp": "Mods/mods.txt and LogicMods, a census of other mods, read only", + "ue_wrap/core/game_build.cpp": "the game's main pak, read only", +} + +# The files that may ask Windows for the exe's path with a null module; all read only. +EXE_PATH_FILES = { + "ue_wrap/core/paths.cpp": "ExeDir itself", + "loader/cppmod_entry.cpp": "the scan for an older mod DLL beside the exe", + "harness/sdk_check.cpp": "the compatibility report's exe line", + "ue_wrap/core/reflection.cpp": "LogGameVersion, the exe's file version", +} + +EXE_DIR_CALL = re.compile(r"\bExeDir\s*\(") +NULL_ARG = r"(?:nullptr|NULL|0)" +EXE_PATH_CALL = re.compile( + r"\bGetModuleFileName[WA]?\s*\(\s*(?:" + NULL_ARG + + r"|(?:::\s*)?GetModuleHandle[WA]?\s*\(\s*" + NULL_ARG + r"\s*\))\s*,") +SUFFIXES = (".cpp", ".h", ".hpp", ".inc") +HEX_DIGITS = frozenset("0123456789abcdefABCDEF") # a set: the empty string at a text edge is not a member + + +def number_run(text, i): + """Whether the word behind text[i] starts with a digit: a number (1, 700, 0xFF, 0xAB'CD'EF; the walk + crosses the earlier separators), where a quote is a separator; case'a' and u8'a' are words, and + their quote opens a character literal.""" + j = i + while j > 0 and (text[j - 1].isalnum() or text[j - 1] in "_'"): + j -= 1 + return text[j:j + 1].isdigit() + + +def strip(text): + """The code of a C++ text, its comments and string and character literals blanked (newlines kept).""" + out, i, n = [], 0, len(text) + while i < n: + c = text[i] + two = text[i:i + 2] + if two == "//": + j = text.find("\n", i) + j = n if j < 0 else j + i = j + elif two == "/*": + j = text.find("*/", i + 2) + j = n if j < 0 else j + 2 + out.append("\n" * text.count("\n", i, j)) + i = j + elif c == "'" and text[i - 1:i] in HEX_DIGITS and text[i + 1:i + 2] in HEX_DIGITS and number_run(text, i): + out.append(c) # a C++14 digit separator (1'700'000'000), not the start of a character literal + i += 1 + elif c == '"' or c == "'": + m = re.match(r'([^()\\ ]{0,16})\(', text[i + 1:i + 19]) if c == '"' and text[i - 1:i] == "R" else None + if m: + end = text.find(")" + m.group(1) + '"', i) + j = n if end < 0 else end + len(m.group(1)) + 2 + else: + j = i + 1 + while j < n and text[j] != c: + j += 2 if text[j] == "\\" else 1 + j = min(n, j + 1) + out.append("\n" * text.count("\n", i, j)) + out.append('""' if c == '"' else "''") + i = j + else: + out.append(c) + i += 1 + return "".join(out) + + +def check(files): + """files: {path relative to src/votv-coop/src, '/'-separated: text}. Returns the faults.""" + faults = [] + for path, text in sorted(files.items()): + code = strip(text) + for m in EXE_DIR_CALL.finditer(code): + if path not in EXE_DIR_FILES: + faults.append("%s:%d: ExeDir( builds a path from the exe's folder; the mod's own data goes " + "through ModDataDir (allowed files: data_paths_gate.EXE_DIR_FILES)" + % (path, code.count("\n", 0, m.start()) + 1)) + for m in EXE_PATH_CALL.finditer(code): + if path not in EXE_PATH_FILES: + faults.append("%s:%d: GetModuleFileName of the null module is the exe's path by another " + "road (allowed files: data_paths_gate.EXE_PATH_FILES)" + % (path, code.count("\n", 0, m.start()) + 1)) + return faults + + +def tree_files(src): + out = {} + for p in sorted(src.rglob("*")): + if p.is_file() and p.suffix in SUFFIXES: + out[p.relative_to(src).as_posix()] = p.read_text(encoding="utf-8", errors="replace") + return out + + +def drill(): + allowed_dir = next(iter(EXE_DIR_FILES)) + allowed_path = next(iter(EXE_PATH_FILES)) + call = "std::wstring p = ue_wrap::paths::ExeDir() + L\"\\\\multivoid.ini\";\n" + cases = { + "ExeDir call in a data builder": {"coop/config/config.cpp": call}, + "ExeDir call over two lines": {"coop/x/a.cpp": "auto d =\n paths::ExeDir\n ();\n"}, + "ExeDir call in a header": {"coop/x/a.h": "inline auto D() { return paths::ExeDir(); }\n"}, + "GetModuleFileNameW(nullptr": {"coop/x/a.cpp": "::GetModuleFileNameW(nullptr, buf, MAX_PATH);\n"}, + "GetModuleFileNameA(NULL": {"coop/x/a.cpp": "GetModuleFileNameA(NULL, buf, n);\n"}, + "GetModuleFileNameW(0": {"coop/x/a.cpp": "GetModuleFileNameW( 0 , buf, n);\n"}, + "GetModuleFileName of GetModuleHandle(nullptr)": { + "coop/x/a.cpp": "::GetModuleFileNameW(::GetModuleHandleW(nullptr), exe, MAX_PATH);\n"}, + "ExeDir after a digit separator": { + "coop/x/a.cpp": "constexpr long k = 1'700'000'000;\nauto d = paths::ExeDir();\n// it's\n"}, + "ExeDir after a 3-group hex literal": { + "coop/x/a.cpp": "auto k = 0xAB'CD'EF; auto d = paths::ExeDir();\n"}, + "ExeDir after a letter-led hex group": { + "coop/x/a.cpp": "auto k = 0xDEAD'BEEF'CAFE; auto d = paths::ExeDir();\n"}, + "ExeDir after case'a'": {"coop/x/a.cpp": "switch (x) { case'a': auto d = paths::ExeDir(); }\n"}, + "ExeDir after u8'a'": {"coop/x/a.cpp": "auto s = u8'a'; auto d = paths::ExeDir();\n"}, + "GetModuleFileName(NULL, over two lines": { + "coop/x/a.cpp": "GetModuleFileNameW(\n NULL,\n buf, n);\n"}, + } + clean = { + "an allowed file calls ExeDir": {allowed_dir: call}, + "an allowed file asks the null module": {allowed_path: "GetModuleFileNameW(nullptr, p, n);\n"}, + "ExeDir in a comment": {"coop/x/a.cpp": "// the old ExeDir() place\n/* ExeDir( */\n"}, + "ExeDir in a string": {"coop/x/a.cpp": 'const char* s = "ExeDir()";\n'}, + "a module variable": {"coop/x/a.cpp": "GetModuleFileNameW(mod, buf, n);\n"}, + "the module's base": {"coop/x/a.cpp": "auto b = ::GetModuleHandleW(nullptr);\n"}, + "GetModuleFileNameEx of a process": { + "coop/x/a.cpp": "K32GetModuleFileNameExW(proc, nullptr, path, MAX_PATH);\n"}, + "a character literal beside a digit separator": { + "coop/x/a.cpp": "char c = 'a'; // ExeDir(\nint n = 0xFF'FF; char d = '\\'';\n"}, + "ModDataDir": {"coop/config/config.cpp": "auto d = ue_wrap::paths::ModDataDir();\n"}, + } + bad = 0 + for name, files in cases.items(): + if not check(files): + print("drill: the gate PASSED a planted fault: %s" % name) + bad += 1 + for name, files in clean.items(): + if check(files): + print("drill: the gate FAILED a clean file: %s" % name) + bad += 1 + print("data_paths_gate drill: %s (%d planted fault(s), %d clean case(s))" + % ("PASS" if not bad else "FAIL", len(cases), len(clean))) + return 1 if bad else 0 + + +def main(): + if "--drill" in sys.argv: + return drill() + base = ROOT + if "--tree" in sys.argv: + base = pathlib.Path(sys.argv[sys.argv.index("--tree") + 1]) + src = base / "src" / "votv-coop" / "src" + if not src.is_dir(): + print("data_paths_gate: FAIL (no source folder %s)" % src) + return 1 + files = tree_files(src) + faults = check(files) + for f in faults: + print(f) + print("data_paths_gate: %s (%d file(s) read, %d fault(s))" + % ("PASS" if not faults else "FAIL", len(files), len(faults))) + return 1 if faults else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/ci/fact_rows_gate.py b/.github/ci/fact_rows_gate.py new file mode 100644 index 000000000..63d665d9a --- /dev/null +++ b/.github/ci/fact_rows_gate.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""fact_rows_gate -- every game-function row of ue_wrap/facts is well formed and has a reference. + +A row (`UE_FACT_FN(ident, "feature", L"class" | nullptr, L"function", Mode)`, declared in the file that +registers it) is what the mod's fingerprint census checks against the reference table of the game +target, `src/votv-coop/include/ue_wrap/facts/fingerprints_.inc` (docs/bytecode-fingerprint.md). +This gate holds, over the mod's source: + + 1. a row lives in a .cpp: in a header it would register once per translation unit; + 2. its class and function are string literals and its feature a lowercase id ([a-z0-9_.]): the + table, this gate and a reader all read them as written; + 3. its mode is one of Declared, Dispatch, AnyClass, RuntimeClass; an AnyClass row has no class, and a + Declared or Dispatch row has one (it is resolved on it, and with none would never install); + 4. its ident is used in its file beyond the declaration: a row nothing registers is a dead fact; + 5. the target's table has an entry for its key -- present or recorded absent for a class row, a + signature set for an any-class row, and for a run-time class row with no class an entry on at + least one class (the census keys it on the class its site resolves) -- so a release never ships a + row the census cannot judge. + + python .github/ci/fact_rows_gate.py # PASS/FAIL over the tree + python .github/ci/fact_rows_gate.py --drill # the gate must fail each planted fault +""" +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +MOD = ROOT / "src" / "votv-coop" +ROW = re.compile(r"UE_FACT_FN\s*\(([^;]*?)\)\s*;", re.S) +ARGS = re.compile(r'^\s*(\w+)\s*,\s*"([^"]*)"\s*,\s*(nullptr|L"[^"]*")\s*,\s*(L"[^"]*")\s*,\s*(\w+)\s*$', re.S) +MODES = {"Declared", "Dispatch", "AnyClass", "RuntimeClass"} +TABLE_FN = re.compile(r'\{L"([^"]*)",\s*L"([^"]*)",') +TABLE_ANY = re.compile(r'\{L"([^"]*)",\s*\d+,\s*\d+,\s*\d+\}') + + +def target(): + m = re.search(r'set\(VOTVCOOP_GAME_TARGET\s+"([^"]+)"\)', (MOD / "CMakeLists.txt").read_text(encoding="utf-8")) + return m.group(1) if m else None + + +def check(files, table_text): + """files: {path: text}. Returns the list of faults.""" + faults = [] + fns = {(c.lower(), f.lower()) for c, f in TABLE_FN.findall(table_text or "")} + anys = {f.lower() for f in TABLE_ANY.findall(table_text or "")} + for path, text in sorted(files.items()): + if path.endswith(("function_row.h",)): + continue # the macro's own definition and its comment example + for m in ROW.finditer(text): + line = text.count("\n", 0, m.start()) + 1 + where = "%s:%d" % (path, line) + a = ARGS.match(m.group(1)) + if not path.endswith(".cpp"): + faults.append("%s: a row outside a .cpp" % where) + if not a: + faults.append("%s: a row whose arguments are not (ident, \"feature\", L\"class\"|nullptr, " + "L\"function\", Mode) literals" % where) + continue + ident, feature, cls, fn, mode = a.groups() + if not re.fullmatch(r"[a-z0-9_.]+", feature): + faults.append("%s: feature %r is not a lowercase id" % (where, feature)) + if mode not in MODES: + faults.append("%s: mode %s is not one of %s" % (where, mode, sorted(MODES))) + if mode == "AnyClass" and cls != "nullptr": + faults.append("%s: an AnyClass row names a class" % where) + if mode in ("Declared", "Dispatch") and cls == "nullptr": + faults.append("%s: a %s row names no class" % (where, mode)) + if len(re.findall(r"\b%s\b" % re.escape(ident), text)) < 2: + faults.append("%s: row %s is declared and never used" % (where, ident)) + if table_text is None: + continue + fname = fn[2:-1].lower() + if mode == "AnyClass": + if fname not in anys: + faults.append("%s: no reference entry for any-class row %s" % (where, fname)) + elif cls == "nullptr": + if fname not in {f for _, f in fns}: + faults.append("%s: no reference entry keys run-time class row %s on any class" % (where, fname)) + elif (cls[2:-1].lower(), fname) not in fns: + faults.append("%s: no reference entry for %s.%s" % (where, cls[2:-1], fn[2:-1])) + return faults + + +def tree_files(): + out = {} + for p in list((MOD / "src").rglob("*")) + list((MOD / "include").rglob("*")): + if p.suffix in (".cpp", ".h", ".inc", ".hpp"): + text = p.read_text(encoding="utf-8", errors="replace") + if "UE_FACT_FN" in text: + out[str(p.relative_to(ROOT)).replace("\\", "/")] = text + return out + + +def drill(): + good_table = ('{L"a_c", L"f", L"a_c.f", 0x1ull, 0x2ull},\n{L"g", 3, 0, 1},\n') + base = 'UE_FACT_FN(kRowF, "lane", L"A_C", L"f", Declared);\nvoid X() { Use(kRowF); }\n' + cases = { + "header row": ({"x/a.h": base}, good_table), + "non-literal class": ({"x/a.cpp": base.replace('L"A_C"', "kClass")}, good_table), + "upper-case feature": ({"x/a.cpp": base.replace('"lane"', '"Lane"')}, good_table), + "unknown mode": ({"x/a.cpp": base.replace("Declared", "Exact")}, good_table), + "unused row": ({"x/a.cpp": base.replace("Use(kRowF);", "")}, good_table), + "no reference": ({"x/a.cpp": base.replace('L"f"', 'L"h"').replace("kRowF", "kRowH")}, good_table), + "any-class with a class": ({"x/a.cpp": 'UE_FACT_FN(kRowG, "lane", L"A_C", L"g", AnyClass);\n' + 'void X() { Use(kRowG); }\n'}, good_table), + "declared row with no class": ({"x/a.cpp": 'UE_FACT_FN(kRowD, "lane", nullptr, L"f", Declared);\n' + 'void X() { Use(kRowD); }\n'}, good_table), + "run-time class row keyed on no class": ({"x/a.cpp": 'UE_FACT_FN(kRowR, "lane", nullptr, L"r", RuntimeClass);\n' + 'void X() { Use(kRowR); }\n'}, good_table), + } + bad = 0 + for name, (files, table) in cases.items(): + if not check(files, table): + print("drill: the gate PASSED a planted fault: %s" % name) + bad += 1 + clean = base + 'UE_FACT_FN(kRowR, "lane", nullptr, L"f", RuntimeClass);\nvoid Y() { Use(kRowR); }\n' + if check({"x/a.cpp": clean}, good_table): + print("drill: the gate FAILED a clean row") + bad += 1 + print("fact_rows_gate drill: %s (%d planted fault(s))" % ("PASS" if not bad else "FAIL", len(cases))) + return 1 if bad else 0 + + +def main(): + if "--drill" in sys.argv: + return drill() + t = target() + table = MOD / "include" / "ue_wrap" / "facts" / ("fingerprints_%s.inc" % t) + table_text = table.read_text(encoding="utf-8") if table.exists() else None + files = tree_files() + faults = check(files, table_text) + if table_text is None: + faults.append("no reference table %s for the game target %s" % (table.relative_to(ROOT), t)) + rows = sum(len(ROW.findall(x)) for p, x in files.items() if not p.endswith("function_row.h")) + for f in faults: + print(f) + print("fact_rows_gate: %s (%d row(s) in %d file(s), reference %s)" % ("PASS" if not faults else "FAIL", rows, + len(files), t)) + return 1 if faults else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/ci/judge.ps1 b/.github/ci/judge.ps1 index 293073228..82709a7a4 100644 --- a/.github/ci/judge.ps1 +++ b/.github/ci/judge.ps1 @@ -134,6 +134,8 @@ if ($tag) { } } else { Add-Check 'FOREIGN_TAG' 'FAIL' 'unevaluable (tag format failed)' } +# The judge's token is read-only, so draft releases are not listed; both checks below hold over +# live releases, which every judge sees. # 10. FOREIGN_RELEASE + COMPLETION (tag-association, R18). $completion = 'NONE' if ($SkipApi) { diff --git a/.github/ci/l10n_pot.py b/.github/ci/l10n_pot.py new file mode 100644 index 000000000..78c95aa07 --- /dev/null +++ b/.github/ci/l10n_pot.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""l10n_pot -- the mod's translation template, and the gate that keeps the call sites honest. + +The template (src/votv-coop/locale/multivoid.pot) is every msgid the mod looks up, extracted from the files that may +hold player text: the allowlist below, kept by hand beside the translation layer's design and never guessed. A +contributor without the maintainer's local tools regenerates it with this script; MTA runs xgettext for the same +job, and ours is a script because the markers inside the config tables and the allowlist are ours. + +Extraction reads C++ as tokens (comments and strings skipped as the compiler skips them), takes `l10n::T`, +`l10n::Tc`, `l10n::Tn`, `L10N_MARK`, `L10N_MARK_C` and `L10N_MARK_N` whose arguments are string literals, joins +adjacent literals and decodes escapes as the compiler does, and writes entries sorted by (msgctxt, msgid) with +file-name references and no line numbers, so the template changes only when a string does. A non-literal argument +(a drawer's `T(entry.name)`) is not extracted: its literal was marked where it is defined. + +Usage: + l10n_pot.py writes the template and the pseudo-locale (locale/en_XA.po) + l10n_pot.py --check the gate: exits 1 on a stale template or pseudo-locale, or on any rule below + +The gate's rules (the translation layer's design, WP-4): + 1. an extraction call or marker with a literal argument in a file outside the allowlist; + 2. an id-bearing ImGui widget (a function of imgui.h whose first parameter is label, name or str_id) whose first + argument is an l10n::T family call rather than an l10n::Label; + 3. an English msgid with a '%' that the conversion grammar refuses, a msgid with two or more conversions that are + not numbered, or a plural pair whose msgids differ in their conversions; + 4. an l10n::Fmt or l10n::FmtSpan call whose format is an l10n::T family call on literals and whose argument count + differs from that msgid's conversions; + 5. a shipped pack (locale/*.po) holding a msgid the template does not. +""" + +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +MOD = ROOT / "src" / "votv-coop" +TEMPLATE = MOD / "locale" / "multivoid.pot" +# The pseudo-locale: every line the English in [[ ]], its conversions intact. Embedded like a real pack, it is how a +# run proves every surface looks its text up -- the l10n drill's translated arm, a screenshot where a line without +# brackets is one that is not -- whatever a real pack covers, and the selftest's read of every embedded pack puts each +# English msgid through the C++ conversion check. Android's tag for the same instrument; a player may pick it. +PSEUDO = MOD / "locale" / "en_XA.po" +IMGUI_H = MOD / "third_party" / "imgui" / "imgui.h" + +# The files that may hold player text (the design's D6), relative to src/votv-coop/. Edited with each call-site +# sheet; a file outside it may not extract. +ALLOWLIST = [ + "include/coop/commands/command_dispatcher.h", + "include/coop/config/config_registry_rows.inc", + "src/coop/bug_report/report_bundle.cpp", + "src/coop/bug_report/report_files.cpp", + "src/coop/bug_report/report_form.cpp", + "src/coop/commands/command_sync.cpp", + "src/coop/comms/peer_action_feed.cpp", + "src/coop/dispatch/event_feed.cpp", + "src/coop/net/end_reason.cpp", + "src/coop/player/local_body.cpp", + "src/coop/player/nameplate.cpp", + "src/coop/player/nick_color.cpp", + "src/coop/player/sleep_sync.cpp", + "src/coop/props/prop_act_request.cpp", + "src/coop/props/remover_request.cpp", + "src/coop/session/player_handshake.cpp", + "src/coop/session/player_handshake_prefs.cpp", + "src/coop/session/player_handshake_version.cpp", + "src/coop/session/server_settings_sync.cpp", + "src/ui/admin_panel.cpp", + "src/ui/browser_input_screens.cpp", + "src/ui/bug_report_pane.cpp", + "src/ui/chat_input.cpp", + "src/ui/config_review_panel.cpp", + "src/ui/console.cpp", + "src/ui/dev_menu.cpp", + "src/ui/end_reason_dialog.cpp", + "src/ui/fonts.cpp", + "src/ui/game_build_dialog.cpp", + "src/ui/hud.cpp", + "src/ui/join_curtain.cpp", + "src/ui/link_format.cpp", + "src/ui/loading_screen.cpp", + "src/ui/net_stats_panel.cpp", + "src/ui/scoreboard.cpp", + "src/ui/server_browser_actions.cpp", + "src/ui/server_browser_native.cpp", + "src/ui/server_browser_panels.cpp", + "src/ui/server_browser_rows.cpp", + "src/ui/server_settings_pane.cpp", + "src/ui/skins_panel.cpp", + "src/ui/voice_icons.cpp", + "src/ui/voice_panel.cpp", + "src/ui/world_rules_panel.cpp", +] +# The layer itself, which looks strings up on purpose and extracts nothing. +EXEMPT_DIRS = ("src/l10n/", "include/l10n/") + +EXTRACT = {"l10n::T": ("id",), "l10n::Tc": ("ctx", "id"), "l10n::Tn": ("id", "plural", None), + "L10N_MARK": ("id",), "L10N_MARK_C": ("ctx", "id"), "L10N_MARK_N": ("id", "plural")} +T_FAMILY = ("l10n::T", "l10n::Tc", "l10n::Tn") + +HEADER = """# The Multivoid interface's translation template. Generated by .github/ci/l10n_pot.py; never edited by hand. +# A translation is a copy of this file named .po (zh_CN.po), with msgstr filled in. +msgid "" +msgstr "" +"Project-Id-Version: Multivoid\\n" +"Content-Type: text/plain; charset=UTF-8\\n" +"Content-Transfer-Encoding: 8bit\\n" +"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n" +""" + + +# ---- tokens ------------------------------------------------------------------------------------------------ + +TOKEN = re.compile(r""" + (?P//[^\n]*|/\*.*?\*/) + | (?P(?:u8|u|U|L)?R"(?P[^()\\ ]{0,16})\((?P.*?)\)(?P=delim)") + | (?P(?:u8|u|U|L)?"(?:[^"\\\n]|\\.)*") + | (?P(?:u8|u|U|L)?'(?:[^'\\\n]|\\.)*') + | (?P[A-Za-z_][A-Za-z_0-9]*(?:::[A-Za-z_][A-Za-z_0-9]*)*) + | (?P[()\[\]{},;]) + | (?P\S) +""", re.S | re.X) + + +def decode(literal): + """A narrow C++ string literal's bytes, as the compiler decodes them (the escapes our sources use).""" + body = literal[literal.index('"') + 1:-1] + out = bytearray() + i = 0 + simple = {"n": 10, "t": 9, "r": 13, "\\": 92, '"': 34, "'": 39, "a": 7, "b": 8, "f": 12, "v": 11, "0": 0} + while i < len(body): + c = body[i] + if c != "\\": + out += c.encode("utf-8") + i += 1 + continue + e = body[i + 1] + if e == "x": + j = i + 2 + while j < len(body) and body[j] in "0123456789abcdefABCDEF": + j += 1 + out.append(int(body[i + 2:j], 16) & 0xFF) + i = j + elif e == "u" or e == "U": + n = 4 if e == "u" else 8 + out += chr(int(body[i + 2:i + 2 + n], 16)).encode("utf-8") + i += 2 + n + elif e in "01234567" and not (e == "0" and (i + 2 >= len(body) or body[i + 2] not in "01234567")): + j = i + 1 + while j < len(body) and j < i + 4 and body[j] in "01234567": + j += 1 + out.append(int(body[i + 1:j], 8) & 0xFF) + i = j + else: + out.append(simple.get(e, ord(e))) + i += 2 + try: + return out.decode("utf-8") + except UnicodeDecodeError: + return None # bytes a test feeds on purpose: never a msgid + + +def tokens(text): + for m in TOKEN.finditer(text): + kind = m.lastgroup + if kind in ("comment",): + continue + if kind == "raw": + yield ("string", None, m.start()) # a raw string is never a msgid here + elif kind == "string": + lit = m.group("string") + yield ("string", None if lit[0] in "uUL" and not lit.startswith("u8") else decode(lit), m.start()) + elif kind == "delim" or kind == "rawbody": + continue + else: + yield (kind, m.group(kind), m.start()) + + +def calls(toks): + """(name, args, position) for every identifier followed by '(', args as token lists split at top-level commas.""" + for i, (kind, val, pos) in enumerate(toks): + if kind != "ident" or i + 1 >= len(toks) or toks[i + 1][1] != "(": + continue + depth, j, args, cur = 0, i + 1, [], [] + while j < len(toks): + k, v, _ = toks[j] + if k == "punct" and v in "([{": + depth += 1 + if depth > 1: + cur.append(toks[j]) + elif k == "punct" and v in ")]}": + depth -= 1 + if depth == 0: + args.append(cur) + break + cur.append(toks[j]) + elif k == "punct" and v == "," and depth == 1: + args.append(cur) + cur = [] + else: + cur.append(toks[j]) + j += 1 + yield val, args, pos, i + + +def literal(arg): + """The joined value of an argument made only of adjacent string literals, else None.""" + if not arg or any(k != "string" for k, _, _ in arg) or any(v is None for _, v, _ in arg): + return None + return "".join(v for _, v, _ in arg) + + +# ---- the conversion grammar (the layer's printf_check, in Python) ------------------------------------------- + +CONV = re.compile(r"%(?:(?P[0-9]{1,3})\$)?(?P[-+ #0]*)(?P[0-9]*)(?:\.(?P[0-9]*))?" + r"(?Pll|l|z)?(?P.)?") + + +def conversions(fmt): + """[(argno, type, length)] or raises ValueError with the reason, for a msgid with a '%'.""" + out, numbered, i = [], None, 0 + while i < len(fmt): + if fmt[i] != "%": + i += 1 + continue + if fmt[i + 1:i + 2] == "%": + i += 2 + continue + m = CONV.match(fmt, i) + t = m.group("type") + flags = m.group("flags") + if t is None: + raise ValueError("a conversion without its type") + if t == "n": + raise ValueError("a '%n' conversion") + if t not in "diuxXofFeEgGs": + raise ValueError("a conversion type outside d i u x X o f F e E g G s") + if len(set(flags)) != len(flags): + raise ValueError("a repeated flag") + if len(m.group("width")) > 2 or (m.group("prec") or "").__len__() > 2: + raise ValueError("a width or precision of more than two digits") + if m.group("prec") is not None and t == "s": + raise ValueError("a precision on %s") + if m.group("len") and t not in "diuxXo": + raise ValueError("a length modifier on a non-integer type") + if "0" in flags and t == "s": + raise ValueError("a '0' flag on %s") + if ("+" in flags or " " in flags) and t not in "difFeEgG": + raise ValueError("a '+' or ' ' flag on an unsigned or string type") + if "#" in flags and t not in "xXofFeEgG": + raise ValueError("a '#' flag on a type it has no form for") + has = m.group("num") is not None + if numbered is None: + numbered = has + elif numbered != has: + raise ValueError("numbered and unnumbered conversions mixed") + arg = int(m.group("num")) if has else len(out) + 1 + out.append((arg, "d" if t == "i" else t, m.group("len") or "")) + i = m.end() + nums = sorted(a for a, _, _ in out) + if nums != list(range(1, len(out) + 1)): + raise ValueError("an argument read twice or a gap in the numbers") + if len(out) > 16: + raise ValueError("more than 16 conversions") + return out + + +def signature(fmt): + return sorted(conversions(fmt)) if "%" in fmt else [] + + +# ---- the scan ------------------------------------------------------------------------------------------------- + +def widget_names(): + # The pinned header is the authority on which calls take a label; without it the gate cannot judge a call site, + # so it refuses rather than read none. + if not IMGUI_H.is_file(): + sys.exit("l10n_pot: %s is absent -- run: git submodule update --init src/votv-coop/third_party/imgui" + % IMGUI_H.relative_to(ROOT).as_posix()) + text = IMGUI_H.read_text(encoding="utf-8", errors="replace") + return set(re.findall(r"IMGUI_API\s+[\w\s\*]*?\b(\w+)\s*\(\s*const\s+char\s*\*\s*(?:label|name|str_id)\b", text)) + + +def scan(): + entries = {} # (ctx, id) -> {"plural": str|None, "refs": set} + problems = [] + widgets = widget_names() + sources = sorted(p for p in MOD.rglob("*") if p.suffix in (".cpp", ".h", ".inc") and "third_party" not in p.parts) + for path in sources: + rel = path.relative_to(MOD).as_posix() + if rel.startswith(EXEMPT_DIRS): + continue + text = path.read_text(encoding="utf-8", errors="replace") + toks = list(tokens(text)) + allowed = rel in ALLOWLIST + for name, args, pos, idx in calls(toks): + line = text.count("\n", 0, pos) + 1 + where = "%s:%d" % (rel, line) + if name in EXTRACT: + roles = EXTRACT[name] + vals = {} + for role, arg in zip(roles, args): + if role: + vals[role] = literal(arg) + if vals.get("id") is None: + continue # a non-literal argument: marked where its literal lives + if not allowed: + problems.append("%s: %s with a literal in a file outside the allowlist" % (where, name)) + continue + key = (vals.get("ctx") or "", vals["id"]) + e = entries.setdefault(key, {"plural": None, "refs": set()}) + e["refs"].add(rel) + if "plural" in vals and vals["plural"] is not None: + e["plural"] = vals["plural"] + try: + a = signature(vals["id"]) + if vals.get("plural") is not None and signature(vals["plural"]) != a: + problems.append("%s: a plural pair whose msgids differ in their conversions" % where) + # A translation may number its conversions only if the msgid does; a msgid with two or + # more is numbered, so a translator can always reorder them. + for m in [vals["id"]] + ([vals["plural"]] if vals.get("plural") else []): + if len(signature(m)) >= 2 and not re.search(r"%[0-9]+\$", m): + problems.append("%s: msgid %r has two or more conversions and no argument numbers" % (where, m)) + except ValueError as err: + problems.append("%s: msgid %r refused by the conversion grammar: %s" % (where, vals["id"], err)) + elif name.split("::")[-1] in widgets and name.startswith("ImGui::") and args: + first = args[0] + if first and first[0][0] == "ident" and first[0][1] in T_FAMILY: + problems.append("%s: %s takes a translated label without l10n::Label" % (where, name)) + elif name in ("l10n::Fmt", "l10n::FmtSpan") and args: + fmt_index = 2 if name == "l10n::Fmt" else 3 + if len(args) <= fmt_index: + continue + f = args[fmt_index] + if not f or f[0][0] != "ident" or f[0][1] not in T_FAMILY: + continue + inner = list(calls(f)) + if not inner: + continue + iname, iargs, _, _ = inner[0] + roles = EXTRACT.get(iname, ()) + msgid = None + for role, arg in zip(roles, iargs): + if role == "id": + msgid = literal(arg) + if msgid is None: + continue + try: + want = len(conversions(msgid)) if "%" in msgid else 0 + except ValueError: + continue # reported where the msgid is extracted + have = len(args) - fmt_index - 1 + if have != want: + problems.append("%s: %s passes %d argument(s) for %r, which names %d" % (where, name, have, msgid, want)) + return entries, problems + + +def quote(s): + s = s.replace("\\", "\\\\").replace('"', '\\"').replace("\t", "\\t").replace("\r", "\\r") + parts = s.split("\n") + if len(parts) == 1: + return '"%s"' % s + lines = [p + "\\n" for p in parts[:-1]] + ([parts[-1]] if parts[-1] else []) + return '""\n' + "\n".join('"%s"' % p for p in lines) + + +def render(entries): + out = [HEADER.rstrip("\n")] + for (ctx, msgid) in sorted(entries): + e = entries[(ctx, msgid)] + out.append("") + out.append("#: " + " ".join(sorted(e["refs"]))) + if "%" in msgid: + out.append("#, c-format") + if ctx: + out.append("msgctxt " + quote(ctx)) + out.append("msgid " + quote(msgid)) + if e["plural"] is not None: + out.append("msgid_plural " + quote(e["plural"])) + out.append('msgstr[0] ""') + out.append('msgstr[1] ""') + else: + out.append('msgstr ""') + return "\n".join(out) + "\n" + + +def render_pseudo(entries): + out = ["# The pseudo-locale en_XA: every line the English in [[ ]]. Generated by .github/ci/l10n_pot.py with the", + "# template; never edited by hand. A test instrument: a line without brackets on screen is not looked up.", + 'msgid ""', 'msgstr ""', + '"Project-Id-Version: Multivoid\\n"', + '"Language: en_XA\\n"', + '"Content-Type: text/plain; charset=UTF-8\\n"', + '"Content-Transfer-Encoding: 8bit\\n"', + '"Plural-Forms: nplurals=2; plural=(n != 1);\\n"'] + for (ctx, msgid) in sorted(entries): + e = entries[(ctx, msgid)] + out.append("") + if "%" in msgid: + out.append("#, c-format") + if ctx: + out.append("msgctxt " + quote(ctx)) + out.append("msgid " + quote(msgid)) + if e["plural"] is not None: + out.append("msgid_plural " + quote(e["plural"])) + out.append("msgstr[0] " + quote("[[" + msgid + "]]")) + out.append("msgstr[1] " + quote("[[" + e["plural"] + "]]")) + else: + out.append("msgstr " + quote("[[" + msgid + "]]")) + return "\n".join(out) + "\n" + + +def pack_msgids(path): + """The (ctx, msgid) pairs a .po file defines, by a minimal scan; validating translations is the runtime's.""" + text = path.read_text(encoding="utf-8", errors="replace") + pairs, ctx, cur, field = set(), "", None, None + def unq(s): + return bytes(s[1:-1], "utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8") + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("msgctxt "): + ctx, field = unq(line[8:]), "ctx" + elif line.startswith("msgid "): + cur, field = unq(line[6:]), "id" + elif line.startswith('"'): + if field == "id": + cur += unq(line) + elif field == "ctx": + ctx += unq(line) + elif line.startswith("msgstr") or line.startswith("msgid_plural"): + if cur is not None and field == "id" and cur != "": + pairs.add((ctx, cur)) + if line.startswith("msgstr"): + ctx = "" + field = None + return pairs + + +def main(): + entries, problems = scan() + text = render(entries) + pseudo = render_pseudo(entries) + if "--check" in sys.argv[1:]: + for path, want in ((TEMPLATE, text), (PSEUDO, pseudo)): + have = path.read_text(encoding="utf-8").replace("\r\n", "\n") if path.exists() else "" + if have != want: + problems.append("%s is stale -- run .github/ci/l10n_pot.py" % path.relative_to(ROOT).as_posix()) + known = set(entries) + for po in sorted((MOD / "locale").glob("*.po")): + for pair in sorted(pack_msgids(po) - known): + problems.append("%s holds %r, which the template does not" % (po.relative_to(ROOT).as_posix(), pair[1])) + for p in problems: + print("l10n_pot: " + p) + print("l10n_pot: %s (%d entries)" % ("FAIL" if problems else "PASS", len(entries))) + return 1 if problems else 0 + if problems: + for p in problems: + print("l10n_pot: " + p) + print("l10n_pot: not written -- fix the problems above") + return 1 + TEMPLATE.parent.mkdir(parents=True, exist_ok=True) + TEMPLATE.write_text(text, encoding="utf-8", newline="\n") + PSEUDO.write_text(pseudo, encoding="utf-8", newline="\n") + print("l10n_pot: wrote %s and %s (%d entries)" % (TEMPLATE.relative_to(ROOT).as_posix(), + PSEUDO.relative_to(ROOT).as_posix(), len(entries))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/ci/l10n_vectors.py b/.github/ci/l10n_vectors.py new file mode 100644 index 000000000..b00aa3b39 --- /dev/null +++ b/.github/ci/l10n_vectors.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""l10n_vectors -- the plural-rule test vectors the l10n selftest checks our evaluator against. + +A catalogue's Plural-Forms header is a C expression over n, and src/l10n/plural_expr.cpp evaluates +it. A vector is (rule, n, the form n takes); a vector whose expected form came from our own +evaluator would pass whatever that evaluator gets wrong, so the forms here come from Python's own +gettext.c2py, an implementation independent of ours. + +The rules are the distinct ones languages ship: the thirteen of MTA's 43 client catalogues (whitespace +aside), plus Polish, Irish and the three-form Czech rule of GNU gettext's manual. They are short +formulas, facts of each language, written here as data. Beside them sit headers the evaluator must +refuse. + +Usage: + l10n_vectors.py writes src/votv-coop/src/l10n/plural_vectors.inc + l10n_vectors.py --check exits 1 when that file differs from what this script writes +""" + +import gettext +import pathlib +import re +import sys + +OUT = pathlib.Path(__file__).resolve().parents[2] / "src" / "votv-coop" / "src" / "l10n" / "plural_vectors.inc" + +RULES = [ + "nplurals=1; plural=0;", + "nplurals=2; plural=(n != 1);", + "nplurals=2; plural=(n > 1);", + "nplurals=2; plural=(n%10==1 && n%100!=11 ? 0 : 1);", + "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);", + "nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);", + "nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;", + "nplurals=3; plural=n==1 ? 0 : n==2 ? 1 : 2;", + "nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2;", + "nplurals=3; plural=(n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2);", + "nplurals=4; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 3;", + "nplurals=4; plural=n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3;", + "nplurals=4; plural=(n%10==1 && (n%100>19 || n%100<11) ? 0 : (n%10>=2 && n%10<=9) && " + "(n%100>19 || n%100<11) ? 1 : n%1!=0 ? 2 : 3);", + "nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && (n%100<12 || n%100>14) ? 1 : " + "n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3);", + "nplurals=4; plural=((n%10==1 && n%100!=11) ? 0 : ((n%10>=2 && n%10<=4 && (n%100<12 || n%100>14)) ? 1 : " + "((n%10==0 || (n%10>=5 && n%10<=9)) || (n%100>=11 && n%100<=14)) ? 2 : 3));", + "nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5);", +] + +N = [0, 1, 2, 3, 4, 5, 7, 11, 12, 14, 21, 22, 25, 100, 101, 102, 111, 1000000, 2 ** 63] + +REFUSED = [ + "nplurals=INTEGER; plural=EXPRESSION;", # a fresh template's placeholder + "nplurals=0; plural=0;", + "nplurals=9; plural=n;", + "nplurals=2;", + "nplurals=2; plural=n ^ 1;", # a token outside the grammar + "nplurals=2; plural=(n != 1;", + "nplurals=2; plural=n != 1 1;", # input left over + "nplurals=2; plural=" + "(" * 33 + "n" + ")" * 33 + ";", # nested deeper than 32 + "nplurals=2; plural=" + "!" * 33 + "n;", + "nplurals=2; plural=" + "n+" * 128 + "n;", # longer than 256 characters + "nplurals=2; plural=12345678901234567890;", # a number of more than 19 digits +] + + +def expression(rule): + m = re.search(r"plural\s*=\s*(.*?);?\s*$", rule) + return m.group(1).rstrip(";") + + +def c_string(s, indent=" ", width=110): + """A C string literal, split into adjacent literals that the compiler joins, to stay in width.""" + body = s.replace("\\", "\\\\").replace('"', '\\"') + room = width - len(indent) - 3 + chunks = [body[i:i + room] for i in range(0, len(body), room)] or [""] + return ("\n" + indent).join('"' + c + '"' for c in chunks) + + +def render(): + lines = [ + "// l10n/plural_vectors.inc -- GENERATED by .github/ci/l10n_vectors.py, never hand-edited. The", + "// forms are Python's gettext.c2py's, an evaluator independent of src/l10n/plural_expr.cpp.", + "", + "constexpr const char* kRules[] = {", + ] + lines += [" " + c_string(r) + "," for r in RULES] + lines += ["};", "", "struct Vector { unsigned char rule; unsigned long long n; unsigned char form; };", + "constexpr Vector kVectors[] = {"] + for i, r in enumerate(RULES): + f = gettext.c2py(expression(r)) + cells = ["{%d, %dull, %d}," % (i, n, f(n)) for n in N] + row = " " + for c in cells: + if len(row) + 1 + len(c) > 110: + lines.append(row) + row = " " + row += " " + c + lines.append(row) + lines += ["};", "", "constexpr const char* kRefusedRules[] = {"] + lines += [" " + c_string(r) + "," for r in REFUSED] + lines += ["};", ""] + return "\n".join(lines) + + +def main(): + text = render() + if "--check" in sys.argv[1:]: + have = OUT.read_text(encoding="utf-8") if OUT.exists() else "" + if have.replace("\r\n", "\n") != text: + print("l10n_vectors: %s is stale -- run .github/ci/l10n_vectors.py" % OUT.name) + return 1 + print("l10n_vectors: %s is current" % OUT.name) + return 0 + OUT.write_text(text, encoding="utf-8", newline="\n") + print("l10n_vectors: wrote %s (%d rules x %d n)" % (OUT.name, len(RULES), len(N))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/ci/ledger_lib.ps1 b/.github/ci/ledger_lib.ps1 index 085af5862..d41d67343 100644 --- a/.github/ci/ledger_lib.ps1 +++ b/.github/ci/ledger_lib.ps1 @@ -7,7 +7,7 @@ Set-StrictMode -Version Latest # ONE tag grammar for the whole lane: v-b[-dev]. # = d.d.d + optional single letter (e.g. 0.9.0n); = decimal, no leading zero. -$script:TagRegex = '^v(?\d+\.\d+\.\d+[a-z]?)-b(?[1-9]\d*)(?-dev)?$' +$script:TagRegex = '^v(?\d+\.\d+\.\d+[a-z]?)-b(?[1-9]\d*)(?-dev)?\z' # \z: '$' would admit one trailing newline # Repo-relative path of the wire-revision header (kProtocolVersion) -- the same # file CMakeLists regex-parses for the build number. @@ -15,6 +15,21 @@ $script:ProtocolHeaderPath = 'src/votv-coop/include/coop/net/protocol.h' function Get-ReleaseTagRegex { $script:TagRegex } +# The commit a tag names, from `git ls-remote --tags` output: the peeled line +# (refs/tags/^{}) of an annotated tag wins over the tag line itself. Ref +# names match exact and case-sensitive, so a near-name (0) never answers. +# $null when neither line is listed. +function Select-TagCommitFromLsRemote { + param([string[]]$Lines, [Parameter(Mandatory)][string]$TagName) + $direct = $null; $peeled = $null + foreach ($line in $Lines) { + $sha, $ref = "$line" -split "`t", 2 + if ($ref -ceq "refs/tags/$TagName") { $direct = $sha } + elseif ($ref -ceq "refs/tags/$TagName^{}") { $peeled = $sha } + } + if ($peeled) { $peeled } else { $direct } +} + function ConvertFrom-ReleaseTag { param([Parameter(Mandatory)][string]$TagName) if ($TagName -cnotmatch $script:TagRegex) { return $null } # -c: the grammar is case-SENSITIVE (PS -match is not) @@ -134,6 +149,30 @@ function Get-NewestPublished { $null } +# The newest PUBLISHED stable per game target -- Get-NewestPublished's fold, per game: a hashtable +# game -> ledger state. The masters' COOP_LATEST_GAMES is this map (docs/release.md), so a player on one game +# is told about the newest Multivoid for that game and not about one for another. +function Get-NewestPublishedPerGame { + param([Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Rows, + [switch]$IncludeDev) + $out = @{} + $cands = @($Rows | Where-Object { $_.Kind -eq 'published' -and ($IncludeDev -or -not $_.Dev) }) + for ($i = $cands.Count - 1; $i -ge 0; $i--) { + if ($out.ContainsKey($cands[$i].Game)) { continue } + $st = Get-LedgerState -Rows $Rows -N $cands[$i].N + if ($st.State -eq 'PUBLISHED') { $out[$cands[$i].Game] = $st } + } + $out +} + +# COOP_LATEST_GAMES's value for a map game -> anything with N: ":: b", comma-joined, oldest N +# first -- the master's grammar (server/src/lobby.rs parse_latest_games). +function Format-LatestGames { + param([Parameter(Mandatory)][hashtable]$PerGame) + (@($PerGame.GetEnumerator() | Sort-Object { [int]$_.Value.N }) | + ForEach-Object { "$($_.Key):$($_.Value.N):$($_.Key) b$($_.Value.N)" }) -join ',' +} + # --- Proto extraction ---------------------------------------------------- # kProtocolVersion at a given commit, read via `git show` (no checkout needed). diff --git a/.github/ci/master_contact_gate.py b/.github/ci/master_contact_gate.py index 6208c7af5..cba08c5c0 100644 --- a/.github/ci/master_contact_gate.py +++ b/.github/ci/master_contact_gate.py @@ -44,7 +44,10 @@ # is downstream of an action the player took; see b349de42's message for the # lane-by-lane reasoning. ALLOWED_HTTP = { - "coop/net/lobby_announcer.cpp": {"/v1/host", "/v1/heartbeat", "/v1/visibility", "/v1/leave"}, + # `/v1/host/challenge` is the nonce a create signs: the same host action as `/v1/host`, one call + # earlier, so it is sent only when the player hosts through a master. + "coop/net/lobby_announcer.cpp": {"/v1/host/challenge", "/v1/host", "/v1/heartbeat", "/v1/visibility", + "/v1/leave"}, # `` is the /v1/lobbies fetch: the path is assembled with query # parameters, so it reaches http::Get as a variable rather than a literal. # Declared explicitly so an unreadable endpoint still has to be admitted. diff --git a/.github/ci/public_prose_baseline.json b/.github/ci/public_prose_baseline.json index a702ffbbb..867fc2abd 100644 --- a/.github/ci/public_prose_baseline.json +++ b/.github/ci/public_prose_baseline.json @@ -2,7 +2,7 @@ "as_of": "860892917", "note": "measured on the working tree; as_of is HEAD at that moment", "counters": { - "md.files": 50, + "md.files": 53, "md.lines": 7696, "md.over_600": 0, "md.cyrillic": 1, diff --git a/.github/ci/publish.ps1 b/.github/ci/publish.ps1 index ffe508754..709bd57b0 100644 --- a/.github/ci/publish.ps1 +++ b/.github/ci/publish.ps1 @@ -22,6 +22,18 @@ $ErrorActionPreference = 'Stop' $tag = ConvertFrom-ReleaseTag $TagName if (-not $tag) { throw "tag '$TagName' fails the grammar (the judge should have refused)" } +if ($TagSha -cnotmatch '^[0-9a-f]{40}\z') { throw "TagSha '$TagSha' is not a full lowercase commit sha" } + +# The release hangs on the tag NAME, its assets on TagSha. The tag is re-read from +# the repository before the draft and again right before the flip makes the release public: +# a tag moved after the judge would publish these assets under another commit. +function Assert-TagStillNamesJudgedSha { + $lines = @(git ls-remote --tags "https://github.com/$Repo.git" "refs/tags/$TagName*") + if ($LASTEXITCODE -ne 0) { throw "re-reading tag '$TagName' from $Repo failed" } + $now = Select-TagCommitFromLsRemote -Lines $lines -TagName $TagName + if ($now -cne $TagSha) { throw "tag '$TagName' now names '$now', not the judged $TagSha -- refusing to publish" } + Write-Host "tag $TagName still names the judged $TagSha" +} # --- Assets: ONE zip, assembled from the artifact payload (UE4SS_ARC 7.4c/8.3) -- # The artifact carries the tagged cacheless rebuild's main.dll; the zip is @@ -116,6 +128,7 @@ if ($shaMatches.Count -ne $assets.Count) { throw "final body carries $($shaMatch $bodyFile = Join-Path ([System.IO.Path]::GetTempPath()) "multivoid-relbody-$($tag.N).md" Set-Content -LiteralPath $bodyFile -Value $body -Encoding utf8 -NoNewline +Assert-TagStillNamesJudgedSha gh release create $TagName --repo $Repo --draft --title $title --notes-file $bodyFile @($assets.FullName) | Out-Host if ($LASTEXITCODE -ne 0) { throw 'draft creation failed' } @@ -132,6 +145,7 @@ foreach ($a in $assets) { } # --- Flip ----------------------------------------------------------------- +Assert-TagStillNamesJudgedSha $flipArgs = @('release', 'edit', $TagName, '--repo', $Repo, '--draft=false') $flipArgs += $(if ($tag.Dev) { '--prerelease' } else { '--prerelease=false' }) & gh @flipArgs | Out-Host @@ -161,6 +175,10 @@ if ($tag.Dev) { Write-Host "published: $title ($TagName @ $TagSha)" Write-Host 'RITUAL REMINDER: verify the release page, then append the published row to .github/ci/LEDGER.tsv (closes state(N) API-free).' if (-not $tag.Dev) { - Write-Host 'STABLE: update EVERY master''s env constants, then run .github/ci/verify_latest.ps1:' - Write-Host " COOP_LATEST_PROTO=$($tag.N) COOP_LATEST_MOD=`"$($tag.Game) b$($tag.N)`"" + # Every game's newest published stable, this one included (its published row is appended after): the whole + # list each time, so a master's line is replaced, never edited. + $perGame = Get-NewestPublishedPerGame -Rows (Read-Ledger -Path (Join-Path $PSScriptRoot 'LEDGER.tsv')).Rows + $perGame[$tag.Game] = [pscustomobject]@{ N = $tag.N } + Write-Host 'STABLE: set EVERY master''s COOP_LATEST_GAMES to this line, then run .github/ci/verify_latest.ps1:' + Write-Host " COOP_LATEST_GAMES=`"$(Format-LatestGames -PerGame $perGame)`"" } diff --git a/.github/ci/registry_gate.ps1 b/.github/ci/registry_gate.ps1 index ae3984b58..f13f07723 100644 --- a/.github/ci/registry_gate.ps1 +++ b/.github/ci/registry_gate.ps1 @@ -44,14 +44,22 @@ $ErrorActionPreference = 'Stop' if (-not $Root) { $Root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) } $incPath = Join-Path $Root 'src\votv-coop\include\coop\config\config_registry_rows.inc' -# The [dev] section's own file, which the row list includes at its end. -$devIncPath = Join-Path $Root 'src\votv-coop\include\coop\config\config_registry_rows_dev.inc' $cfgHdr = Join-Path $Root 'src\votv-coop\include\coop\config\config.h' $srcDirs = @((Join-Path $Root 'src\votv-coop\src'), (Join-Path $Root 'src\votv-coop\include')) -# The registry's own files self-reference every ident (decl/def/table, the -# rows and their [dev] file) and must not satisfy the census. -$selfFiles = @('config_registry.h', 'config_registry.cpp', 'config_registry_rows.inc', - 'config_registry_rows_dev.inc') +# The row files: the list and every row file it includes, recursively, as the compiler reads them (the [dev] +# section's file, and the files that section is split into), so a split needs no change here. +function Get-RowFiles([string]$path) { + $files = @($path) + foreach ($line in (Get-Content $path)) { + if ($line -match '^\s*#include\s+"coop/config/(?config_registry_rows[^"]*\.inc)"') { + $files += Get-RowFiles (Join-Path (Split-Path -Parent $path) $Matches['f']) + } + } + return $files +} +# The registry's own files self-reference every ident (decl/def/table, and the +# row files, added as they are read) and must not satisfy the census. +$selfFiles = @('config_registry.h', 'config_registry.cpp') $violations = New-Object System.Collections.Generic.List[string] $controlFailures = New-Object System.Collections.Generic.List[string] @@ -72,8 +80,8 @@ $writeOnlyAllow = @{ if (-not (Test-Path $incPath)) { Write-Host "registry_gate: FAIL -- missing $incPath"; exit 1 } $rowIdents = New-Object System.Collections.Generic.List[string] $fontRoleIdents = New-Object System.Collections.Generic.List[string] -$rowFiles = @($incPath) -if (Test-Path $devIncPath) { $rowFiles += $devIncPath } +$rowFiles = @(Get-RowFiles $incPath) +$selfFiles += @($rowFiles | ForEach-Object { Split-Path -Leaf $_ }) foreach ($line in ($rowFiles | ForEach-Object { Get-Content $_ })) { if ($line -cmatch '^\s*CFG_(?[A-Z_]+)\(\s*(?[A-Za-z0-9_]+)\s*,') { # ROWFLAGS and LABEL mark a row declared above it (config_registry_rows.inc); neither is a row. diff --git a/.github/ci/release_sign.py b/.github/ci/release_sign.py index e65f261ce..d28076cce 100644 --- a/.github/ci/release_sign.py +++ b/.github/ci/release_sign.py @@ -10,7 +10,8 @@ build it belongs to and an Ed25519 signature over `signed_message`. `verify` applies the same grammar and steps as the game, and prints which step refused. `keygen` makes a key in memory, hands its private half straight to the `release` environment's secret through `gh`, and prints only the -table row; it refuses first unless that environment is protected as docs/release.md sets it. +table row; it refuses first unless that environment, and the rulesets over the refs it admits, +are set as docs/release.md sets them. The game's check is the vendored ed25519-donna `ed25519_sign_open` (src/votv-coop/third_party/GameNetworkingSockets/src/external/ed25519-donna/ed25519.c); `verify` is the same cofactorless, byte-exact check (RFC 8032 section 5.1.7 allows it): it @@ -31,7 +32,7 @@ environment only, and no message and no exception text this file writes holds it. Exit codes: 0 done; 1 verification failed; 2 a usage or key-table error; 3 the environment (the -seed, or `gh` and the repository's `release` environment). +seed, or `gh`, the repository's `release` environment and its rulesets). Every message is one line on stdout and starts `release_sign: `. """ import argparse @@ -426,6 +427,53 @@ def _check_protection(environment, policies): raise _unprotected("branch policies") +# The release environment admits the v* tags and main, so whoever may create or move such a tag, +# or push to main, reaches the key. Only an admin may: for each requirement an active ruleset over +# that ref with no exclusion, refusing the rule to all but the admin repository role (actor 5, as +# the live rulesets read). The rulesets are read in list order and any one may hold a requirement. +_ADMIN_ROLE = ("RepositoryRole", 5) +_REQUIREMENTS = (("tag", "refs/tags/v*", "creation"), ("tag", "refs/tags/v*", "update"), + ("tag", "refs/tags/v*", "deletion"), ("branch", "refs/heads/main", "update")) + + +def _unruled(pattern, rule): + return SignError("protection", 3, "release_sign: the refs the release environment admits are " + "not held by rulesets as docs/release.md sets it (%s %s)" % (pattern, rule)) + + +def _ruleset_holds(ruleset, pattern, rule): + """Whether one ruleset refuses `rule` on `pattern` to all but the admin repository role.""" + conditions = ruleset.get("conditions") if isinstance(ruleset, dict) else None + names = conditions.get("ref_name") if isinstance(conditions, dict) else None + if not (isinstance(names, dict) and isinstance(names.get("include"), list) + and pattern in names["include"] and names.get("exclude") == []): + return False + rules = ruleset.get("rules") + kinds = {r.get("type") for r in rules if isinstance(r, dict)} if isinstance(rules, list) else set() + if rule not in kinds: + return False + bypass = ruleset.get("bypass_actors") + return isinstance(bypass, list) and all( + isinstance(a, dict) and (a.get("actor_type"), a.get("actor_id")) == _ADMIN_ROLE + for a in bypass) + + +def _check_ref_rulesets(runner, repo): + listed = _gh_api(runner, "repos/%s/rulesets?per_page=100" % repo) + read = [] + for summary in listed if isinstance(listed, list) else []: + if not (isinstance(summary, dict) and summary.get("enforcement") == "active" + and summary.get("target") in ("tag", "branch")): + continue + ruleset_id = summary.get("id") + if isinstance(ruleset_id, int) and not isinstance(ruleset_id, bool): + read.append((summary["target"], + _gh_api(runner, "repos/%s/rulesets/%d" % (repo, ruleset_id)))) + for target, pattern, rule in _REQUIREMENTS: + if not any(t == target and _ruleset_holds(r, pattern, rule) for t, r in read): + raise _unruled(pattern, rule) + + def _key_id(text): """The strict parse `sign --build` applies to its number: plain ASCII digits, no sign, no underscore, no leading zero.""" @@ -435,8 +483,9 @@ def _key_id(text): def cmd_keygen(args, runner, out): - """The order is the custody: refuse an unprotected environment, then make the key, check its - row, hand the seed to gh's standard input, and print the row. No file is written.""" + """The order is the custody: refuse an unprotected environment or unheld release refs, then + make the key, check its row, hand the seed to gh's standard input, and print the row. No file + is written.""" if _REPO.fullmatch(args.repo) is None: raise SignError("usage", 2, "release_sign: --repo must be owner/name") rows = _table_rows() @@ -449,6 +498,7 @@ def cmd_keygen(args, runner, out): environment = _gh_api(runner, base) policies = _gh_api(runner, base + "/deployment-branch-policies") _check_protection(environment, policies) + _check_ref_rulesets(runner, args.repo) seed = os.urandom(32) pub = ed25519_public(seed) diff --git a/.github/ci/release_sign_keygen_drill.py b/.github/ci/release_sign_keygen_drill.py index 897bd917f..898aca521 100644 --- a/.github/ci/release_sign_keygen_drill.py +++ b/.github/ci/release_sign_keygen_drill.py @@ -3,11 +3,11 @@ Standard library only. A fake runner answers canned `gh` output and records every call's argv and keyword arguments, so no process starts and the real `gh` is never run; each run happens inside a -temporary working directory with stdout and stderr captured. The fake answers only the three -commands keygen may run, with the output a real `gh` prints (so an echo of it is seen), and acts -on the keyword arguments the way `subprocess.run` does. Three mutants are always run and the same -assertion helper as the green arms must flag each of them; one starts a process, which the guard -below must refuse. +temporary working directory with stdout and stderr captured. The fake answers only the commands +keygen may run (its reads, then the secret), with the output a real `gh` prints (so an echo of it +is seen), and acts on the keyword arguments the way `subprocess.run` does. Three mutants are +always run and the same assertion helper as the green arms must flag each of them; one starts a +process, which the guard below must refuse. The drill names a repository that does not exist and, for its whole run, refuses the process routes it can see with an AssertionError, so a bypass of the injected runner through one of them @@ -61,6 +61,17 @@ REPO = "drill-owner/drill-repo" ENV_ARGV = ["gh", "api", "repos/%s/environments/release" % REPO] POLICIES_ARGV = ["gh", "api", "repos/%s/environments/release/deployment-branch-policies" % REPO] +RULESETS_ARGV = ["gh", "api", "repos/%s/rulesets?per_page=100" % REPO] +BRANCH_RULESET_ID, TAG_RULESET_ID, MAIN_RULESET_ID = 10, 11, 12 +DETAIL_PREFIX = "repos/%s/rulesets/" % REPO + + +def ruleset_argv(ruleset_id): + return ["gh", "api", DETAIL_PREFIX + str(ruleset_id)] + + +READS = [ENV_ARGV, POLICIES_ARGV, RULESETS_ARGV, ruleset_argv(BRANCH_RULESET_ID), + ruleset_argv(TAG_RULESET_ID), ruleset_argv(MAIN_RULESET_ID)] SECRET_ARGV = ["gh", "secret", "set", rs.SEED_ENV, "--env", "release", "--repo", REPO] API_KWARGS = {"capture_output": True, "text": True, "encoding": "utf-8"} SECRET_KWARGS = dict(API_KWARGS, errors="replace") @@ -98,6 +109,55 @@ def canned_policies(*extra): {"id": 3 + i, "node_id": "n%d" % i, "name": n, "type": t} for i, (n, t) in enumerate(items)]} +ADMIN_BYPASS = {"actor_id": 5, "actor_type": "RepositoryRole", "bypass_mode": "always"} +WRITE_BYPASS = {"actor_id": 4, "actor_type": "RepositoryRole", "bypass_mode": "always"} +# id -> (target, name, ref pattern, rule types, bypass actors): the live repository's three +# rulesets, renumbered 10 to 12 in its list order, and a second v* ruleset (13) for the arm that +# reads two. +RULESET_SHAPES = { + BRANCH_RULESET_ID: ("branch", "main-no-force-no-delete", "refs/heads/main", + ["deletion", "non_fast_forward"], []), + TAG_RULESET_ID: ("tag", "v-tags-admin-only", "refs/tags/v*", + ["creation", "update", "deletion"], [ADMIN_BYPASS]), + MAIN_RULESET_ID: ("branch", "main-push-admin-only", "refs/heads/main", ["update"], + [ADMIN_BYPASS]), + 13: ("tag", "v-tags-second", "refs/tags/v*", ["creation", "update", "deletion"], + [ADMIN_BYPASS]), +} + + +def dropping_absent(values): + return {k: v for k, v in values.items() if v is not ABSENT} + + +def canned_rulesets(ids=(BRANCH_RULESET_ID, TAG_RULESET_ID, MAIN_RULESET_ID), edit=None, drop=()): + """The repository's ruleset list, in the order of `ids` less `drop`; `edit` maps an id to + overrides of its summary.""" + items = [] + for ruleset_id in ids: + if ruleset_id in drop: + continue + target, name = RULESET_SHAPES[ruleset_id][:2] + summary = {"id": ruleset_id, "name": name, "target": target, "enforcement": "active", + "source_type": "Repository"} + summary.update((edit or {}).get(ruleset_id, {})) + items.append(summary) + return items + + +def canned_details(edit=None): + """GitHub's REST answer for each ruleset, by id, shaped as the live repository's; `edit` + maps an id to overrides (a value of ABSENT drops that key).""" + details = {} + for ruleset_id, (target, name, pattern, rules, bypass) in RULESET_SHAPES.items(): + detail = {"id": ruleset_id, "name": name, "target": target, "enforcement": "active", + "conditions": {"ref_name": {"exclude": [], "include": [pattern]}}, + "rules": [{"type": t} for t in rules], "bypass_actors": list(bypass)} + detail.update((edit or {}).get(ruleset_id, {})) + details[ruleset_id] = dropping_absent(detail) + return details + + class Uncanned(Exception): """The code under test ran a command the fake has no answer for.""" @@ -105,9 +165,11 @@ class Uncanned(Exception): class FakeGh: """A runner that answers from canned values and records every call; it starts no process.""" - def __init__(self, environment=None, policies=None, secret=None): + def __init__(self, environment=None, policies=None, secret=None, rulesets=None, details=None): self.environment = canned_env() if environment is None else environment self.policies = canned_policies() if policies is None else policies + self.rulesets = canned_rulesets() if rulesets is None else rulesets + self.details = canned_details() if details is None else details self.secret = secret or (lambda text: (0, GH_LINE, GH_WARNING)) self.calls = [] @@ -119,6 +181,12 @@ def __call__(self, argv, **kwargs): answer = self.environment elif argv == POLICIES_ARGV: answer = self.policies + elif argv == RULESETS_ARGV: + answer = self.rulesets + elif (len(argv) == 3 and argv[:2] == ["gh", "api"] and argv[2].startswith(DETAIL_PREFIX) + and argv[2][len(DETAIL_PREFIX):].isdigit() + and int(argv[2][len(DETAIL_PREFIX):]) in self.details): + answer = self.details[int(argv[2][len(DETAIL_PREFIX):])] else: raise Uncanned(" ".join(argv)) if isinstance(answer, (dict, list)): @@ -215,9 +283,10 @@ def no_process(): rs.subprocess, rs.os = real_sub, real_os -def keygen_flaws(keygen, fake, table, key_id=2, seeds=None): +def keygen_flaws(keygen, fake, table, key_id=2, seeds=None, reads=READS): """Every custody rule a green keygen run must keep; an empty list is a clean run. `key_id` is - the id the run asks for; the seed each run hands to gh is appended to `seeds`.""" + the id the run asks for; the seed each run hands to gh is appended to `seeds`; `reads` is the + exact list of reads the run makes before the secret.""" table_before = table.read_bytes() r = drive(keygen, fake, table, key_id=str(key_id)) flaws = [] @@ -230,11 +299,11 @@ def need(name, ok): need("no exception (%s)" % r["raised"], r["raised"] is None) need("exit 0", r["rc"] == 0) need("stderr empty", r["stderr"] == "") - need("three calls, the secret last", - len(calls) == 3 and len(secrets) == 1 and calls[2] is secrets[0]) - need("the two protection reads, exact argv and kwargs", - [c[0] for c in calls[:2]] == [ENV_ARGV, POLICIES_ARGV] - and all(c[1] == API_KWARGS for c in calls[:2])) + last = len(reads) + need("%d calls, the secret last" % (last + 1), + len(calls) == last + 1 and len(secrets) == 1 and calls[last] is secrets[0]) + need("the protection reads, exact argv and kwargs", + [c[0] for c in calls[:last]] == reads and all(c[1] == API_KWARGS for c in calls[:last])) need("the dictated gh secret set argv", bool(secrets) and secrets[0][0] == SECRET_ARGV) need("the secret set kwargs, exactly, and the input", bool(secrets) and "input" in secrets[0][1] @@ -315,8 +384,21 @@ def drill_keygen_green(tmp): ("--id 254 through main", via_main, FakeGh(), 254)): flaws = keygen_flaws(keygen, fake, table, key_id, seeds) check("keygen green with " + name + (": " + ", ".join(flaws) if flaws else ""), not flaws) + excluded = {"conditions": {"ref_name": {"exclude": ["refs/tags/v0*"], + "include": ["refs/tags/v*"]}}} + two_tag_rulesets = FakeGh(rulesets=canned_rulesets(ids=(10, 11, 12, 13)), + details=canned_details({TAG_RULESET_ID: excluded})) + flaws = keygen_flaws(rs.cmd_keygen, two_tag_rulesets, table, 2, seeds, + reads=READS + [ruleset_argv(13)]) + check("keygen green with two tag rulesets, the first listed flawed and the second holding" + + (": " + ", ".join(flaws) if flaws else ""), not flaws) + # No bypass at all holds a requirement too: nobody, admins included, may move the ref. + no_bypass = FakeGh(details=canned_details({TAG_RULESET_ID: {"bypass_actors": []}})) + flaws = keygen_flaws(rs.cmd_keygen, no_bypass, table, 2, seeds) + check("keygen green with an empty bypass list on the tag ruleset" + + (": " + ", ".join(flaws) if flaws else ""), not flaws) check("keygen green: every run's seed is its own and none is the public selftest seed", - len(seeds) == 10 and len(set(seeds)) == len(seeds) and rs.TEST_SEED_HEX not in seeds) + len(seeds) == 12 and len(set(seeds)) == len(seeds) and rs.TEST_SEED_HEX not in seeds) def drill_keygen_mutants(tmp): @@ -361,6 +443,46 @@ def drill_keygen_refuses(tmp): {"id": 4, "node_id": "n4", "name": "main", "type": "branch"}]}, "branch policies"), "an environment that is not an object": ([1, 2], None, "admins can bypass"), } + # Each: (the ruleset list or None, edits to the ruleset details or None, the requirement the + # refusal must name). The first requirement a ruleset breaks is the tag's creation. + creation = ("refs/tags/v*", "creation") + main_update = ("refs/heads/main", "update") + tag_rules = lambda *kinds: {TAG_RULESET_ID: {"rules": [{"type": k} for k in kinds]}} + tag_bypass = lambda *actors: {TAG_RULESET_ID: {"bypass_actors": list(actors)}} + unruled = { + "no tag ruleset": (canned_rulesets(drop=(TAG_RULESET_ID,)), None, creation), + "the tag ruleset only evaluated": (canned_rulesets( + edit={TAG_RULESET_ID: {"enforcement": "evaluate"}}), None, creation), + "a ruleset list that is not a list": ({"id": 11}, None, creation), + "another tag pattern": (None, {TAG_RULESET_ID: {"conditions": {"ref_name": { + "exclude": [], "include": ["refs/tags/release-*"]}}}}, creation), + "an exclusion": (None, {TAG_RULESET_ID: {"conditions": {"ref_name": { + "exclude": ["refs/tags/v0*"], "include": ["refs/tags/v*"]}}}}, creation), + "no conditions": (None, {TAG_RULESET_ID: {"conditions": None}}, creation), + "creation allowed": (None, tag_rules("update", "deletion"), creation), + "update allowed": (None, tag_rules("creation", "deletion"), ("refs/tags/v*", "update")), + "deletion allowed": (None, tag_rules("creation", "update"), ("refs/tags/v*", "deletion")), + "a write role may bypass": (None, tag_bypass(ADMIN_BYPASS, WRITE_BYPASS), creation), + "an OrganizationAdmin with id 5": (None, tag_bypass( + dict(ADMIN_BYPASS, actor_type="OrganizationAdmin")), creation), + "a bypass list that is absent": (None, {TAG_RULESET_ID: {"bypass_actors": ABSENT}}, + creation), + "a boolean id on the tag summary": (canned_rulesets(edit={TAG_RULESET_ID: {"id": True}}), + None, creation), + "a string id on the tag summary": (canned_rulesets(edit={TAG_RULESET_ID: {"id": "11"}}), + None, creation), + # The v* rules on a ruleset whose target is a branch hold no tag: the target is checked. + "the v* rules on a branch ruleset": (canned_rulesets( + edit={TAG_RULESET_ID: {"target": "branch"}}), {TAG_RULESET_ID: {"target": "branch"}}, + creation), + "main pushes open": (canned_rulesets(drop=(MAIN_RULESET_ID,)), None, main_update), + "main's ruleset lets a write role bypass": (None, {MAIN_RULESET_ID: { + "bypass_actors": [ADMIN_BYPASS, WRITE_BYPASS]}}, main_update), + "main's rules on a tag ruleset": (canned_rulesets(edit={MAIN_RULESET_ID: {"target": "tag"}}), + None, main_update), + } + unruled_text = ("release_sign: the refs the release environment admits are not held by " + "rulesets as docs/release.md sets it (%s %s)") gh_failed = { "the environment missing": (FakeGh(environment=(1, "", "gh: Not Found (HTTP 404)\nmore")), "release_sign: gh api failed: gh: Not Found (HTTP 404)"), @@ -368,9 +490,16 @@ def drill_keygen_refuses(tmp): "release_sign: gh is not installed or cannot run"), "a non-JSON answer": (FakeGh(policies=(0, "", "")), "release_sign: gh api failed (the answer is not JSON)"), + "the ruleset list read failed": (FakeGh(rulesets=(1, "", "gh: Not Found (HTTP 404)")), + "release_sign: gh api failed: gh: Not Found (HTTP 404)"), + "a ruleset read failed": (FakeGh(details={ + **canned_details(), BRANCH_RULESET_ID: (1, "", "gh: Not Found (HTTP 404)")}), + "release_sign: gh api failed: gh: Not Found (HTTP 404)"), } cases = [(name, FakeGh(env, pol), UNPROTECTED % check_name) for name, (env, pol, check_name) in refusals.items()] + cases += [(name, FakeGh(rulesets=lst, details=canned_details(edit)), unruled_text % required) + for name, (lst, edit, required) in unruled.items()] cases += [(name, fake, text) for name, (fake, text) in gh_failed.items()] for name, fake, text in cases: r = drive(rs.cmd_keygen, fake, table) @@ -385,7 +514,7 @@ def drill_keygen_secret(tmp): r = drive(rs.cmd_keygen, fake, table) check("keygen secret fails: the message is dictated and holds nothing else", r["rc"] == 3 and r["out"] == "release_sign: gh secret set failed (exit 1)\n" - and r["stderr"] == "" and len(fake.calls) == 3) + and r["stderr"] == "" and len(fake.calls) == 7) fake = FakeGh(secret=lambda text: FileNotFoundError("gh")) r = drive(rs.cmd_keygen, fake, table) check("keygen secret set cannot run: exit 3", @@ -408,7 +537,7 @@ def loses_the_new_row(text): rs.read_rows = original check("keygen row read back: a row that does not is exit 3 and no secret is set", r["rc"] == 3 and r["out"] == "release_sign: the new row did not read back\n" - and r["stderr"] == "" and not fake.secret_calls() and len(fake.calls) == 2 + and r["stderr"] == "" and not fake.secret_calls() and len(fake.calls) == 6 and r["left"] == []) diff --git a/.github/ci/reliablekind_gate.py b/.github/ci/reliablekind_gate.py index c81982243..c21a88110 100644 --- a/.github/ci/reliablekind_gate.py +++ b/.github/ci/reliablekind_gate.py @@ -39,12 +39,20 @@ parser cannot read, or any file it cannot open. It fails CLOSED: a gate that cannot read its subject must not return PASS. + 5. The receive edge no longer asking the authorship table, or no longer refusing Host and NotAKind with a + return before the save-transfer diversion and the inbox; or the table (`client_authorship.h`, which + the host's receive edge reads to refuse a kind a client may not author) out of step with the enum: an + enumerator with no row, a row named twice, a row that is not Client or Host, a row for a name that is + not a kind, or a relayable kind that is not Client (the edge would refuse what the relay forwards). + WHAT IT DELIBERATELY DOES NOT HOLD. `session_lanes.h`'s three switches -- `LaneForKind`, `IsClientRelayableReliableKind`, `IsPreWorldSendableKind` -- all default safely (Normal / no-relay / hold-until-world-ready), and whether a given kind SHOULD relay is a design decision no text scan can make. Absence there is legitimate for every host-authored kind. So they are reported in `--table`, not gated: this file must not manufacture a failure it -cannot justify, or the next author learns to route around it. +cannot justify, or the next author learns to route around it. The authorship table is the +exception, by design: it is total -- every kind decided, Client or Host -- and only its +membership is checked here, never whether a decision is right. Note that membership in `session_lanes.h` is NOT receiver coverage. That is not a detail: the 2026-09-01 dead kind WAS in that header (79 case labels), so a gate written around @@ -65,6 +73,14 @@ PROTO = "include/coop/net/protocol.h" LANES = "src/coop/net/session_lanes.h" SESSION = "src/coop/net/session_receive.cpp" # the receive switch and the admission handler +AUTH = "src/coop/net/client_authorship.h" # which kinds a client may author (rule 5) +# The receive edge's question, which is the only reason the table matters. +EDGE_READS_TABLE = re.compile(r"AuthorshipOf\(\s*static_cast\(\s*rh\.kind\s*\)\s*\)") +# What a kind's row may say; NotAKind is the fallback for a byte that names no kind, never a row. +ROW_VALUES = ("Client", "Host") +# The edge's frame: the host-only wrapper, the table's answer named, and the test that opens the refusal. +EDGE_SHAPE = re.compile(r"if\s*\(\s*cfg_\.role\s*==\s*Role::Host\s*\)\s*\{\s*const\s+ClientAuthorship\s+(\w+)\s*=\s*" + r"AuthorshipOf\(\s*static_cast\(\s*rh\.kind\s*\)\s*\)\s*;\s*if\s*\(") # The five family dispatchers. Each Handle*Event returns true iff msg.kind is in its # family, so its own switch IS the membership declaration (SyncRouter consolidation, @@ -187,7 +203,7 @@ def strip_comments(text: str) -> str: def read_tree() -> dict: """Every file the gate reasons about, as text. One dict so --drill can mutate a copy.""" - want = {PROTO, LANES, SESSION, FEED_FILE} | set(FAMILY_FILES) + want = {PROTO, LANES, SESSION, FEED_FILE, AUTH} | set(FAMILY_FILES) for spec in NET_THREAD_TERMINAL.values(): want |= {f for f, _rx, _what in spec["must"]} tree = {} @@ -200,6 +216,77 @@ def read_tree() -> dict: return tree +def matching(src: str, i: int) -> int: + """src[i] opens a bracket: the index just past the bracket that closes it, or -1.""" + pair = {"(": ")", "{": "}"}[src[i]] + depth = 0 + for j in range(i, len(src)): + if src[j] == src[i]: + depth += 1 + elif src[j] == pair: + depth -= 1 + if depth == 0: + return j + 1 + return -1 + + +def edge_findings(src: str) -> list: + """The receive edge, comment-stripped, read as code: on the host only, the table's answer is tested at once + for exactly Host or NotAKind, the block that test opens ends in a return, and all of it comes before the + save-transfer diversion and the inbox. Tokens alone are not enough: a removed Host test once left + `ClientAuthorship::Host` standing in the log line's ternary.""" + m = EDGE_READS_TABLE.search(src) + if not m: + return [f"{SESSION} no longer asks AuthorshipOf about a received kind: the table below would " + f"refuse nothing, however complete it is."] + after = [src.find(tok, m.end()) for tok in ("ReliableKind::SaveTransferChunk", "reliableInbox_.emplace_back")] + if min(after) < 0: + return [f"{SESSION}: the AuthorshipOf check no longer precedes the save-transfer diversion and the " + f"inbox, so a refused kind would be acted on first."] + shape = EDGE_SHAPE.search(src) + if not shape or not shape.start() < m.start() < shape.end(): + return [f"{SESSION}: the edge is no longer `if (cfg_.role == Role::Host) {{ const ClientAuthorship who = " + f"AuthorshipOf(...); if (...)`: only the host asks, and it tests the answer at once."] + who = shape.group(1) + cond_end = matching(src, shape.end() - 1) + cond = " ".join(src[shape.end():cond_end - 1].split()) + want = f"{who} == ClientAuthorship::Host || {who} == ClientAuthorship::NotAKind" + out = [] + if cond != want: + out.append(f"{SESSION}: the edge's test is `{cond}`, not `{want}`: a Host kind and a byte naming no " + f"kind are refused, and nothing else is.") + body_start = cond_end + len(src[cond_end:]) - len(src[cond_end:].lstrip()) + body_end = matching(src, body_start) if src[body_start:body_start + 1] == "{" else -1 + if body_end < 0 or not re.search(r"\breturn\s*;\s*$", src[body_start + 1:body_end - 1]): + out.append(f"{SESSION}: the edge no longer returns on a refusal, so a refused kind goes on to the " + f"inbox and the relay.") + elif body_end > min(after): + out.append(f"{SESSION}: the edge's refusal block runs past the save-transfer diversion or the inbox.") + elif not src[body_end:].lstrip().startswith("}"): + out.append(f"{SESSION}: the host-only block holds more than the table's question and its refusal, " + f"so what follows the refusal there runs on every message a client sends.") + return out + + +def parse_authorship(text: str): + """client_authorship.h's AuthorshipOf: [(kind, row)] in source order, one per case label.""" + src = strip_comments(text) + i = src.find("AuthorshipOf(") + if i < 0: + return None + rows, pending = [], [] + for line in src[i:].splitlines(): + for k in CASE.findall(line): + pending.append(k) + r = re.search(r"return\s+ClientAuthorship::(\w+)\s*;", line) + if r and pending: + rows.extend((k, r.group(1)) for k in pending) + pending = [] + if re.match(r"^\}", line): + break + return rows + + def parse_enum(text): """Return (name -> value-or-None, [unreadable rows]). @@ -325,6 +412,33 @@ def idof(n): f"whether the file still MENTIONS the kind is not enough -- both " f"these files name these kinds in unrelated places.") + # --- check 5: the authorship table is total and in step ------------------------ + fails.extend(edge_findings(strip_comments(tree[SESSION]))) + rows = parse_authorship(tree[AUTH]) + if rows is None: + fails.append(f"{AUTH}: no AuthorshipOf switch found -- the receive edge's table cannot be checked.") + else: + seen = {} + for k, row in rows: + seen.setdefault(k, []).append(row) + for k in sorted(set(seen) - set(enums)): + fails.append(f"{AUTH} has a row for '{k}', which is not a ReliableKind.") + for k in sorted(set(enums), key=lambda x: (enums.get(x) is None, enums.get(x) or 0)): + if k not in seen: + fails.append(f"{k} = {idof(k)} has no row in {AUTH}: decide whether a client may author it " + f"(Client) or not (Host). A kind with no row is refused at the host's edge.") + elif len(seen[k]) > 1: + fails.append(f"{k} = {idof(k)} is named {len(seen[k])} times in {AUTH}.") + for k, rs in sorted(seen.items()): + if rs[0] not in ROW_VALUES: + fails.append(f"{k}'s row in {AUTH} is {rs[0]}; a kind's row is one of " + f"{', '.join(ROW_VALUES)}.") + relay = parse_lane_switches(tree[LANES])["relay"] + for k in sorted(relay): + if k in seen and seen[k][0] != "Client": + fails.append(f"{k} is relayable but {seen[k][0]} in {AUTH}: the edge would refuse what the " + f"relay forwards.") + # --- check 4: two families claiming one kind ----------------------------------- for name in sorted(set(enums), key=lambda k: (enums.get(k) is None, enums.get(k) or 0)): who = [pathlib.PurePosixPath(rel).name for rel, got in fam.items() if name in got] @@ -462,7 +576,7 @@ def drill() -> int: # 3. a family stops being chained: every case label survives, the call does not. t = dict(base) - t[FEED_FILE] = t[FEED_FILE].replace("if (HandleWorldEvent(session, msg)) break;", "", 1) + t[FEED_FILE] = t[FEED_FILE].replace("if (HandleWorldEvent(msg)) break;", "", 1) controls.append(("a family dispatcher no longer chained", t, "HandleWorldEvent is defined in")) @@ -484,6 +598,50 @@ def drill() -> int: t[PROTO] = "// the enum is gone" controls.append(("an unparseable enum", t, "expected >=100")) + # 7-10. the authorship table (rule 5). + row = lambda k: " case ReliableKind::" + k + ":\n" + t = dict(base) + t[AUTH] = t[AUTH].replace(row("ChatMessage"), "", 1) + controls.append(("an authorship row removed", t, "ChatMessage = ")) + t = dict(base) + t[AUTH] = t[AUTH].replace(row("ItemActivate"), "", 1).replace( + row("PropConvert"), row("PropConvert") + row("ItemActivate"), 1) + controls.append(("a relayable kind made Host", t, "ItemActivate is relayable but Host")) + t = dict(base) + t[SESSION] = t[SESSION].replace("AuthorshipOf(static_cast(rh.kind))", "ClientAuthorship::Client", 1) + controls.append(("the receive edge no longer reads the table", t, "no longer asks AuthorshipOf")) + t = dict(base) + t[SESSION] = t[SESSION].replace("RefusalTally::Clock::now());\n }\n return;\n", + "RefusalTally::Clock::now());\n }\n", 1) + controls.append(("the receive edge no longer returns on a refusal", t, "no longer returns on a refusal")) + t = dict(base) + t[SESSION] = t[SESSION].replace("who == ClientAuthorship::NotAKind", "false", 1) + controls.append(("the receive edge accepts a byte naming no kind", t, "the edge's test is")) + t = dict(base) + t[SESSION] = t[SESSION].replace("who == ClientAuthorship::Host || who == ClientAuthorship::NotAKind", + "who == ClientAuthorship::NotAKind", 1) + controls.append(("the receive edge accepts a host-only kind", t, "the edge's test is")) + t = dict(base) + t[SESSION] = t[SESSION].replace("who == ClientAuthorship::NotAKind)", + "who == ClientAuthorship::NotAKind && false)", 1) + controls.append(("the receive edge's test can never be true", t, "the edge's test is")) + t = dict(base) + t[SESSION] = t[SESSION].replace("if (cfg_.role == Role::Host) {\n const ClientAuthorship who", + "if (cfg_.role == Role::Client) {\n const ClientAuthorship who", 1) + controls.append(("the receive edge asks on the wrong peer", t, "the edge is no longer")) + t = dict(base) + t[SESSION] = t[SESSION].replace( + " }\n }\n // payloadLen is a uint16", + " }\n return;\n }\n // payloadLen is a uint16", 1) + controls.append(("a return after the refusal ends every message", t, "holds more than the table's")) + for value in ("NotAKind", "Unverified"): + t = dict(base) + t[AUTH] = t[AUTH].replace(row("ChatMessage"), "", 1).replace( + " return ClientAuthorship::Host;\n }\n", + " return ClientAuthorship::Host;\n" + row("ChatMessage") + + " return ClientAuthorship::" + value + ";\n }\n", 1) + controls.append(("a kind's row is " + value, t, "ChatMessage's row in")) + bad = 0 for label, mutated, expect in controls: got, _e, _f = check(mutated) diff --git a/.github/ci/tag_regex_selftest.ps1 b/.github/ci/tag_regex_selftest.ps1 index 25e190c7e..a35732a75 100644 --- a/.github/ci/tag_regex_selftest.ps1 +++ b/.github/ci/tag_regex_selftest.ps1 @@ -2,7 +2,8 @@ # 2026-07-26 release-body/notes/install gates): MUST-MATCH rows with expected # parse fields + MUST-REFUSE near-twins, for the tag grammar AND the notes # format lint AND the install-doc staleness patterns AND the body writer's -# machine-key invariants. Runs in every release judge pass and locally. +# machine-key invariants, and the pick of a tag's commit from ls-remote lines. +# Runs in every release judge pass, in the repo gates, and locally. # Exit 0 = all rows hold; exit 1 = a grammar/gate drifted. Set-StrictMode -Version Latest @@ -34,7 +35,8 @@ $mustRefuse = @( 'v0.9.0n-b012', # leading zero 'v0.9.0n-b0', # zero build 'v0.9.0n-b12-DEV', # case matters - 'v0.9.0n-b12 ' # trailing whitespace + 'v0.9.0n-b12 ', # trailing whitespace + "v0.9.0n-b12`n" # trailing newline ('$' would admit it, '\z' does not) ) foreach ($t in $mustRefuse) { $p = ConvertFrom-ReleaseTag $t @@ -53,6 +55,17 @@ Assert-Case (@(Test-ReleaseNotesFormat -Content "## What's new`n- x").Count -gt Assert-Case (@(Test-ReleaseNotesFormat -Content "- fine`nsource: 0123456789abcdef0123456789abcdef01234567").Count -gt 0) 'notes: source:-grammar line refused (first-match shadowing)' Assert-Case (@(Test-ReleaseNotesFormat -Content ("- fine`nsha256: " + ('a' * 64) + ' x.dll')).Count -gt 0) 'notes: sha256:-grammar line refused' +# --- Tag-commit pick (publish's re-read of the tag on origin) ---------------- +# The pure function over `git ls-remote --tags` lines: a lightweight tag is its +# own line; an annotated one adds a peeled ^{} line whose sha is the commit. +$shaA = 'a' * 40; $shaB = 'b' * 40; $shaC = 'c' * 40 +Assert-Case ((Select-TagCommitFromLsRemote -Lines @("$shaA`trefs/tags/v1.0.0-b5") -TagName 'v1.0.0-b5') -ceq $shaA) 'pick: a lightweight tag names its own sha' +Assert-Case ((Select-TagCommitFromLsRemote -Lines @("$shaB`trefs/tags/v1.0.0-b5", "$shaA`trefs/tags/v1.0.0-b5^{}") -TagName 'v1.0.0-b5') -ceq $shaA) 'pick: an annotated tag names the peeled commit' +Assert-Case ((Select-TagCommitFromLsRemote -Lines @("$shaA`trefs/tags/v1.0.0-b5^{}", "$shaB`trefs/tags/v1.0.0-b5") -TagName 'v1.0.0-b5') -ceq $shaA) 'pick: the peeled line wins whichever line comes first' +Assert-Case ((Select-TagCommitFromLsRemote -Lines @("$shaC`trefs/tags/v1.0.0-b50", "$shaA`trefs/tags/v1.0.0-b5") -TagName 'v1.0.0-b5') -ceq $shaA) 'pick: a near-name line (b50) beside b5 is ignored' +Assert-Case ($null -eq (Select-TagCommitFromLsRemote -Lines @("$shaC`trefs/tags/v1.0.0-b50", "$shaC`trefs/tags/V1.0.0-B5") -TagName 'v1.0.0-b5')) 'pick: no line for the tag -> $null (a near-name or other case never answers)' +Assert-Case ($null -eq (Select-TagCommitFromLsRemote -Lines @() -TagName 'v1.0.0-b5')) 'pick: empty input -> $null' + # --- Body-writer machine-key invariants: BOTH artifact eras ------------------ # The writer's era switch is data-driven (what the sha map holds), so both eras # get fixtures: the zip era is what publish emits from WP-2 commit 3 onward; the diff --git a/.github/ci/verify_latest.ps1 b/.github/ci/verify_latest.ps1 index bae36cc42..2ae742010 100644 --- a/.github/ci/verify_latest.ps1 +++ b/.github/ci/verify_latest.ps1 @@ -1,5 +1,6 @@ -# verify_latest.ps1 -- the STABLE-ritual closing check (design D3): after every master's env -# constants are updated, each master's /v1/latest must report exactly the newest published stable. +# verify_latest.ps1 -- the STABLE-ritual closing check (design D3): after every master's COOP_LATEST_GAMES is +# updated, each master's /v1/latest must list every game's newest published stable under `games`, and its +# legacy `proto` must be the 0.9.0n entry's -- the game every client older than `games` targets. # The masters are the official slots in protocol.h (kOfficialMasterSlots): a player reads the update # check from the master they chose, so one master left behind tells its players a wrong verdict. # Fold-aware (R23): "the newest BARE-tag row whose state(N) == PUBLISHED" -- a retracted N has a @@ -35,10 +36,14 @@ if ($MasterLatestUrls.Count -eq 0) { } $kind = if ($AllowDev) { 'published (dev admitted)' } else { 'published stable' } -$expected = Get-NewestPublished -Rows (Read-Ledger -Path $LedgerPath).Rows -IncludeDev:$AllowDev -if ($null -ne $expected) { - Write-Host "verify_latest: ledger newest $kind = N=$($expected.N) game=$($expected.Game) tag=$($expected.TagName)" +$rows = (Read-Ledger -Path $LedgerPath).Rows +$perGame = Get-NewestPublishedPerGame -Rows $rows -IncludeDev:$AllowDev +foreach ($g in $perGame.Keys) { + Write-Host "verify_latest: ledger newest $kind for $g = N=$($perGame[$g].N) tag=$($perGame[$g].TagName)" } +# The legacy fields answer clients older than `games`, all made for this game (server/src/lobby.rs LEGACY_GAME). +$legacyGame = '0.9.0n' +$expected = if ($perGame.ContainsKey($legacyGame)) { $perGame[$legacyGame] } else { $null } $failed = 0 foreach ($url in $MasterLatestUrls) { @@ -52,21 +57,40 @@ foreach ($url in $MasterLatestUrls) { $masterProto = 0 if ($resp -and ($resp.PSObject.Properties.Name -contains 'proto')) { $masterProto = [int]$resp.proto } + Write-Host "verify_latest: $url reports proto=$masterProto mod='$($resp.mod)'" + $hint = if ($AllowDev) { '' } else { ' -- if the masters were deliberately pointed at a DEV prerelease, re-run with -AllowDev' } + # The legacy pair: the legacy game's newest, or no record when the ledger has none for it. if ($null -eq $expected) { - if ($masterProto -le 0) { - Write-Host "verify_latest: OK_EMPTY -- no $kind in the ledger; $url has no released record (proto<=0)" - continue + if ($masterProto -gt 0) { + Write-Host "verify_latest: FAIL -- $url reports proto=$masterProto but the ledger has NO $kind for $legacyGame (unrecorded release?)$hint" + $failed++ } - $hint = if ($AllowDev) { '' } else { ' -- if the masters were deliberately pointed at a DEV prerelease, re-run with -AllowDev' } - Write-Host "verify_latest: FAIL -- $url reports proto=$masterProto but the ledger has NO $kind (unrecorded release?)$hint" + } elseif ($masterProto -ne $expected.N) { + Write-Host "verify_latest: FAIL -- $url legacy proto $masterProto != ledger N $($expected.N), the newest $kind for $legacyGame (COOP_LATEST_GAMES not updated / stale?)" $failed++ - continue } - - Write-Host "verify_latest: $url reports proto=$masterProto mod='$($resp.mod)'" - if ($masterProto -ne $expected.N) { - Write-Host "verify_latest: FAIL -- $url proto $masterProto != ledger N $($expected.N) from the newest $kind (env constants not updated / stale?)" - $failed++ + # Every game: each one the ledger has, at its newest; none the ledger does not. + $listed = @{} + if ($resp.PSObject.Properties.Name -contains 'games') { + foreach ($e in @($resp.games)) { $listed[[string]$e.game] = [int]$e.proto } + } + foreach ($g in $perGame.Keys) { + if (-not $listed.ContainsKey($g)) { + Write-Host "verify_latest: FAIL -- $url lists no release for $g under games (ledger N $($perGame[$g].N))" + $failed++ + } elseif ($listed[$g] -ne $perGame[$g].N) { + Write-Host "verify_latest: FAIL -- $url lists $g at N=$($listed[$g]), ledger N $($perGame[$g].N)" + $failed++ + } + } + foreach ($g in $listed.Keys) { + if (-not $perGame.ContainsKey($g)) { + Write-Host "verify_latest: FAIL -- $url lists $g at N=$($listed[$g]) but the ledger has NO $kind for it (unrecorded release?)$hint" + $failed++ + } + } + if ($perGame.Count -eq 0 -and $listed.Count -eq 0 -and $masterProto -le 0) { + Write-Host "verify_latest: OK_EMPTY -- no $kind in the ledger; $url has no released record" } } diff --git a/.github/ci/workflow_expr_gate.py b/.github/ci/workflow_expr_gate.py new file mode 100644 index 000000000..99c00381c --- /dev/null +++ b/.github/ci/workflow_expr_gate.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""workflow_expr_gate -- no ${{ }} expression is pasted into a run: or script: value. + +GitHub substitutes an expression into a run: script (and into actions/github-script's +script: JS) as TEXT before the shell or the interpreter parses it, so a value that carries +quotes or $( ) becomes code: a tag name, a dispatch input or a job output turns into a +command in whatever step holds it. A value reaches a script through the step's env: and is +read as a variable ($env:NAME, "$NAME", process.env.NAME), where it stays data. + +FAILS on any ${{ inside such a value, in .github/workflows/*.yml and in composite actions: +the rest of the key's line plus every following line that is blank or indented deeper (a +block, a folded scalar, an anchor or tag before it, a quoted scalar over lines, a value on +the next line), a quoted key, a flow mapping. A comment inside the script counts too. +Every run first fires two fixtures (a violating workflow must fail, a clean one must pass); +if either does not hold, the gate cannot be trusted and exits 2. + +USAGE + python .github/ci/workflow_expr_gate.py +""" +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT_KEY = re.compile(r"^(?P\s*(?:-\s+)?)[\"']?(?:run|script)[\"']?\s*:(?P.*)$") +FLOW_KEY = re.compile(r"[{,]\s*[\"']?(?:run|script)[\"']?\s*:") + + +def violations(text): + """(line number, line) of every ${{ inside a run:/script: value of one YAML file.""" + found = [] + column = None + for no, line in enumerate(text.splitlines(), 1): + if column is not None: + if not line.strip() or len(line) - len(line.lstrip()) > column: + if "${{" in line: + found.append((no, line.strip())) + continue + column = None + m = SCRIPT_KEY.match(line) + if m: + column = len(m.group("lead")) + if "${{" in m.group("rest"): + found.append((no, line.strip())) + continue + flow = FLOW_KEY.search(line) + if flow and "${{" in line[flow.end():]: + found.append((no, line.strip())) + return found + + +FIXTURE_BAD = """jobs: + a: + steps: + - name: inline + run: echo "${{ inputs.x }}" + - shell: pwsh + run: | + $t = '${{ github.event.inputs.tag }}' + # ${{ secrets.K }} in a comment is substituted too + env: + OK: ${{ inputs.y }} + - name: value on the next line + run: + echo ${{ a }} + - name: quoted scalar over two lines + run: "echo + ${{ a }}" + - name: space before the colon + run : echo ${{ a }} + - name: quoted key + "run": echo ${{ b }} + - name: tag before the block + run: !!str | + echo ${{ a }} + - name: anchor before the block + run: &x | + echo ${{ a }} + - {name: n, run: "echo ${{ c }}"} + - uses: actions/github-script@0 + with: + script: | + const t = '${{ inputs.tag }}'; +""" +FIXTURE_BAD_LINES = [5, 8, 9, 14, 17, 19, 21, 24, 27, 28, 32] +FIXTURE_GOOD = """jobs: + a: + runs-on: ${{ matrix.os }} + steps: + - name: env passes the value + if: ${{ inputs.go }} + env: + TAG: ${{ inputs.tag }} + run: | + Write-Host "tag $env:TAG" + - name: value on the next line, read through env + env: + X: ${{ inputs.x }} + run: + Write-Host $env:X + - name: a dedent ends the value + run: | + Write-Host ok + if: ${{ x }} + - uses: actions/cache@0 + with: + key: k-${{ hashFiles('x') }} +""" + + +def fixtures_hold(): + bad = [no for no, _ in violations(FIXTURE_BAD)] + good = violations(FIXTURE_GOOD) + if bad != FIXTURE_BAD_LINES: + print(f"workflow_expr_gate: FIXTURE the violating workflow fired on lines {bad}, expected {FIXTURE_BAD_LINES}") + return False + if good: + print(f"workflow_expr_gate: FIXTURE the clean workflow fired: {good}") + return False + return True + + +def main(): + if not fixtures_hold(): + return 2 + files = sorted((ROOT / ".github" / "workflows").glob("*.y*ml")) + files += sorted((ROOT / ".github" / "actions").glob("**/action.y*ml")) + if not files: + print("workflow_expr_gate: FAIL no workflow found under .github/workflows") + return 2 + bad = 0 + for path in files: + for no, line in violations(path.read_text(encoding="utf-8")): + rel = path.relative_to(ROOT).as_posix() + print(f"{rel}:{no}: an expression inside a run: script -- pass it through env: ({line})") + bad += 1 + if bad: + print(f"workflow_expr_gate: FAIL {bad} expression(s) inside run: scripts") + return 1 + print(f"workflow_expr_gate: PASS {len(files)} workflow files, no expression inside a run: script") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build-core.yml b/.github/workflows/build-core.yml index d8043e379..6cea97626 100644 --- a/.github/workflows/build-core.yml +++ b/.github/workflows/build-core.yml @@ -7,6 +7,8 @@ # contrib branch shadow this file while the caller's diff reads clean (R22). # Changes here land only as human commits on main; this file's sha256 is part # of the release fingerprint (.github/ci/fingerprint.json, design D3/R23). +# No ${{ }} expression is pasted into a run: script (workflow_expr_gate.py): +# a script reads its values through env:. # Build steps based on PR #2 by huoyan1231 (reworked per design D2). name: build-core @@ -53,34 +55,25 @@ jobs: with: ref: ${{ inputs.ref }} submodules: false + persist-credentials: false # Full clone: submodule-pinned commits + vcpkg versions DB need history. fetch-depth: 0 - name: Self-check -- tree is the requested source (R27) shell: pwsh + env: + SOURCE_SHA: ${{ inputs.source-sha }} run: | $head = (git rev-parse HEAD).Trim() Write-Host "HEAD = $head" - Write-Host "source-sha = ${{ inputs.source-sha }}" - Write-Host "github.sha = ${{ github.sha }} (evidence only -- on a recovery dispatch this is main's HEAD while the tree is the tag's)" - if ($head -ne '${{ inputs.source-sha }}') { throw "checkout mismatch: HEAD $head != source-sha ${{ inputs.source-sha }}" } + Write-Host "source-sha = $env:SOURCE_SHA" + Write-Host "github.sha = $env:GITHUB_SHA (evidence only -- on a recovery dispatch this is the dispatched ref's head, main by the ritual, while the tree is the tag's)" + if ($head -ne $env:SOURCE_SHA) { throw "checkout mismatch: HEAD $head != source-sha $env:SOURCE_SHA" } - name: Init build submodules (third_party only; reference/* never fetched) shell: pwsh run: git submodule update --init --recursive src/votv-coop/third_party - - name: Ensure opus submodule present (BUILDING.md troubleshooting fallback) - # Old clones lack the opus .gitmodules entry; --recursive then skips - # the gitlink silently and configure fails. - shell: pwsh - run: | - $opus = 'src/votv-coop/third_party/opus' - if (-not (Test-Path (Join-Path $opus 'CMakeLists.txt'))) { - Write-Host 'opus submodule empty -> cloning v1.5.2 directly' - Remove-Item -Recurse -Force $opus -ErrorAction SilentlyContinue - git clone --depth 1 -b v1.5.2 https://github.com/xiph/opus.git $opus - } else { Write-Host 'opus present, ok' } - - name: Config registry gate (arc 3 C5 -- dead rows / write-only / alias / ratchet surface) # Standing gate with fixture-injected must-fire controls each run; the # script exits explicitly (pwsh last-child-code lesson). Cheap text @@ -232,8 +225,10 @@ jobs: - name: Install vcpkg (clone full + checkout the baseline pin + bootstrap) shell: pwsh + env: + PIN: ${{ steps.baseline.outputs.sha }} run: | - $pin = '${{ steps.baseline.outputs.sha }}' + $pin = $env:PIN if (-not (Test-Path "$env:VCPKG_ROOT\.git")) { git clone https://github.com/microsoft/vcpkg.git $env:VCPKG_ROOT } @@ -271,16 +266,18 @@ jobs: - name: Configure (CMake + vcpkg manifest mode) shell: pwsh + env: + GENERATOR: ${{ steps.vs.outputs.generator }} run: | - cmake -S src/votv-coop -B ${{ env.BUILD_DIR }} ` - -G "${{ steps.vs.outputs.generator }}" -A x64 ` - -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" ` - -DVCPKG_TARGET_TRIPLET=${{ env.VCPKG_TRIPLET }} ` + cmake -S src/votv-coop -B $env:BUILD_DIR ` + -G $env:GENERATOR -A x64 ` + -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" ` + -DVCPKG_TARGET_TRIPLET="$env:VCPKG_TRIPLET" ` -DVCPKG_MANIFEST_MODE=ON - name: Build (Release) shell: pwsh - run: cmake --build ${{ env.BUILD_DIR }} --config Release + run: cmake --build $env:BUILD_DIR --config Release - name: ABI gate (D-3 -- the built DLL imports NOTHING from UE4SS) # The C-ABI slim contract's whole payoff is a number: 0 imported UE4SS @@ -296,23 +293,20 @@ jobs: # silently discarded the drill's failure -- the script runs the drill # first and returns 2 immediately if it cannot fire. shell: pwsh - run: python .github/ci/abi_gate.py --drill "${{ env.BUILD_DIR }}/Release/main.dll" + run: python .github/ci/abi_gate.py --drill "$env:BUILD_DIR/Release/main.dll" - - name: Package the r2modman zip + verify it (CI evidence, pak-less by construction) + - name: Package the r2modman zip + verify it (CI evidence) # THE DISTRIBUTABLE IS ASSEMBLED ON EVERY BUILD RUN, not first at release time. # `package.ps1` is the same script the release lane runs, and it verifies the # zip it just wrote with the predicate the drill above proved can fail -- so a # change that quietly breaks the package tree (a moved payload, a manifest key, # an identity parse) fails HERE, on the commit that caused it. # - # LAWFULLY PAK-LESS. `assets/paks/` holds untracked binaries and does not exist - # in a CI checkout; package.ps1 documents that a CI zip is pak-less and that a - # pak-less package is still a fully working mod (client_model falls back to the - # game's own skin). A RELEASE zip is assembled on a box where that dir is - # stocked, so this step proves the STRUCTURE, never the payload set. + # The starter paks are tracked in `assets/paks/`, so this zip carries the same + # paks a release zip does; package.ps1 stages them by default. shell: pwsh run: | - pwsh -File .github/ci/package.ps1 -BuildDir "${{ env.BUILD_DIR }}/Release" ` + pwsh -File .github/ci/package.ps1 -BuildDir "$env:BUILD_DIR/Release" ` -OutDir "$env:RUNNER_TEMP/package" exit $LASTEXITCODE @@ -322,7 +316,7 @@ jobs: run: | $out = Join-Path $env:RUNNER_TEMP 'artifact' New-Item -ItemType Directory $out | Out-Null - $dlls = @(Get-ChildItem "${{ env.BUILD_DIR }}/Release/*.dll") + $dlls = @(Get-ChildItem "$env:BUILD_DIR/Release/*.dll") if ($dlls.Count -eq 0) { throw 'no DLLs produced' } foreach ($d in $dlls) { Copy-Item $d.FullName $out diff --git a/.github/workflows/release-core.yml b/.github/workflows/release-core.yml index 0a19dac78..4d96bfa11 100644 --- a/.github/workflows/release-core.yml +++ b/.github/workflows/release-core.yml @@ -1,10 +1,15 @@ -# release-core -- the release lane's brain, ALWAYS executed from main -# (the trampoline calls it @main; the recovery dispatch runs main's YAML by -# construction). THIN ORCHESTRATOR (design D3): every refusal / lint / -# predicate decision lives in .github/ci/*.ps1 executed from the MAIN -# checkout (checkout#2) -- the self-check below guards the JUDGE, not just the -# evidence. The run logs workflow_ref/workflow_sha as evidence (not a gate -- -# a gate cannot guard itself; residual D7(a)). +# release-core -- the release lane's brain, executed from main: the +# trampoline calls it @main. A recovery dispatch runs the YAML of the ref it +# is dispatched on, which the ritual dispatches on main; the release +# environment admits only main and the v* tags, both admin-only. +# THIN ORCHESTRATOR (design D3): every refusal / lint / predicate decision +# lives in .github/ci/*.ps1 executed from the MAIN checkout (checkout#2) -- +# the self-check below guards the JUDGE, not just the evidence. The run logs +# workflow_ref/workflow_sha as evidence (not a gate -- a gate cannot guard +# itself; residual D7(a)). +# No ${{ }} expression is pasted into a run: script (workflow_expr_gate.py): +# every value a script reads comes through env:, and the tag name is held to +# the release grammar before any command sees it. name: release-core on: @@ -29,8 +34,10 @@ concurrency: group: release # full serialization; a queued twin lands on ALREADY_PUBLISHED cancel-in-progress: false +# Read-only for every job but publish, the one that writes the Release API (D7). +# The rulesets let only an admin create, move or delete a v* tag or push to main. permissions: - contents: write # Release API only (D7); the v* tag ruleset blocks robot tags independently + contents: read jobs: judge: @@ -46,6 +53,7 @@ jobs: with: ref: main fetch-depth: 0 + persist-credentials: false - name: Self-check -- judge tree is origin/main HEAD shell: pwsh @@ -56,19 +64,25 @@ jobs: $main = (git rev-parse origin/main).Trim() Write-Host "judge HEAD = $head" Write-Host "origin/main = $main" - Write-Host "workflow_ref = ${{ github.workflow_ref }} (evidence)" - Write-Host "workflow_sha = ${{ github.workflow_sha }} (evidence)" + Write-Host "workflow_ref = $env:GITHUB_WORKFLOW_REF (evidence)" + Write-Host "workflow_sha = $env:GITHUB_WORKFLOW_SHA (evidence)" if ($head -ne $main) { throw "judge checkout is not origin/main HEAD (main moved mid-run?) -- re-run" } - - name: Resolve tag + - name: Resolve tag (held to the release grammar before any command reads it) id: resolve shell: pwsh + env: + DISPATCH_TAG: ${{ inputs.tag }} + CALLER_TAG: ${{ inputs.tag-name }} + CALLER_SHA: ${{ inputs.tag-sha }} run: | - $tag = if ('${{ github.event_name }}' -eq 'workflow_dispatch') { '${{ inputs.tag }}' } else { '${{ inputs.tag-name }}' } + . ./.github/ci/ledger_lib.ps1 + $tag = if ($env:GITHUB_EVENT_NAME -eq 'workflow_dispatch') { $env:DISPATCH_TAG } else { $env:CALLER_TAG } if (-not $tag) { throw 'no tag input' } + if (-not (ConvertFrom-ReleaseTag $tag)) { throw 'the tag input fails the release tag grammar' } $sha = (git rev-list -n1 $tag).Trim() if (-not $sha) { throw "tag '$tag' not found on origin" } - Write-Host "tag $tag -> $sha (caller evidence: '${{ inputs.tag-sha }}')" + Write-Host "tag $tag -> $sha (caller evidence: '$env:CALLER_SHA')" "tag-name=$tag" | Out-File -Append $env:GITHUB_OUTPUT "tag-sha=$sha" | Out-File -Append $env:GITHUB_OUTPUT @@ -87,8 +101,10 @@ jobs: shell: pwsh env: GH_TOKEN: ${{ github.token }} + TAG_NAME: ${{ steps.resolve.outputs.tag-name }} + TAG_SHA: ${{ steps.resolve.outputs.tag-sha }} run: | - pwsh -File .github/ci/judge.ps1 -TagName '${{ steps.resolve.outputs.tag-name }}' -TagShaEvidence '${{ steps.resolve.outputs.tag-sha }}' + pwsh -File .github/ci/judge.ps1 -TagName $env:TAG_NAME -TagShaEvidence $env:TAG_SHA $code = $LASTEXITCODE # Explicit exits: without them the step ends with the child's # $LASTEXITCODE (10) and the no-op verdict turned the job RED -- @@ -120,6 +136,8 @@ jobs: needs: [judge, build] if: needs.judge.outputs.verdict == 'publish' runs-on: windows-latest + permissions: + contents: write # The release key is this environment's secret. The job waits here for the reviewer's # approval, and only after it can read the secret; the first step fails the job when the # environment holds none, which is after the build and the approval, not at once. @@ -148,6 +166,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.judge.outputs.tag-sha }} + persist-credentials: false # Only the *.ps1 predicates and the release signer are overlaid, and only from main. A tag can be # created by anyone who can push one; the refuse-to-publish logic must not @@ -178,18 +197,20 @@ jobs: env: GH_TOKEN: ${{ github.token }} MULTIVOID_RELEASE_KEY: ${{ secrets.MULTIVOID_RELEASE_KEY }} + TAG_NAME: ${{ needs.judge.outputs.tag-name }} + TAG_SHA: ${{ needs.judge.outputs.tag-sha }} run: | - pwsh -File .github/ci/publish.ps1 ` - -TagName '${{ needs.judge.outputs.tag-name }}' ` - -TagSha '${{ needs.judge.outputs.tag-sha }}' ` - -ArtifactDir _release_artifact + pwsh -File .github/ci/publish.ps1 -TagName $env:TAG_NAME -TagSha $env:TAG_SHA -ArtifactDir _release_artifact Get-Content _release_artifact/fingerprint-dump.json | Out-Host - name: Summary shell: pwsh + env: + TAG_NAME: ${{ needs.judge.outputs.tag-name }} + TAG_SHA: ${{ needs.judge.outputs.tag-sha }} run: | - "## Release ${{ needs.judge.outputs.tag-name }}" | Out-File -Append $env:GITHUB_STEP_SUMMARY - "source: ${{ needs.judge.outputs.tag-sha }}" | Out-File -Append $env:GITHUB_STEP_SUMMARY + "## Release $env:TAG_NAME" | Out-File -Append $env:GITHUB_STEP_SUMMARY + "source: $env:TAG_SHA" | Out-File -Append $env:GITHUB_STEP_SUMMARY "Ritual: verify the release page, then append the 'published' ledger row (see publish log; STABLE also: master env + verify_latest.ps1)." | Out-File -Append $env:GITHUB_STEP_SUMMARY already-published: diff --git a/.github/workflows/repo-gates.yml b/.github/workflows/repo-gates.yml index 414b9e306..87900f646 100644 --- a/.github/workflows/repo-gates.yml +++ b/.github/workflows/repo-gates.yml @@ -39,6 +39,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + persist-credentials: false - name: Public-leak gate drill run: python3 .github/ci/public_leak_gate_drill.py @@ -113,6 +114,10 @@ jobs: shell: pwsh run: ./.github/ci/ledger_lint.ps1 -SkipApi + - name: Release tag grammar selftest (the tag grammar, the notes and body formats, the tag pick) + shell: pwsh + run: ./.github/ci/tag_regex_selftest.ps1 + # The SOURCE invariants, which need no compiler and so do not belong behind a # twenty-minute Windows build. They lived only inside build-core once, and while the build # lane was manual a prose sweep deleted every `// GEN:` annotation in the net layer -- @@ -142,5 +147,35 @@ jobs: - name: Master-contact gate (nothing tells an outside server where a player is, unasked) run: python3 .github/ci/master_contact_gate.py + - name: Workflow expression gate (a run script reads its values through env, never a pasted expression) + run: python3 .github/ci/workflow_expr_gate.py + - name: ReliableKind routing gate (every wire kind reaches a receiver, or names why not) run: python3 .github/ci/reliablekind_gate.py + + - name: Fact rows gate drill + run: python3 .github/ci/fact_rows_gate.py --drill + + - name: Fact rows gate (every watched or hooked game function is a well-formed row with a reference) + run: python3 .github/ci/fact_rows_gate.py + + - name: Data paths gate drill + run: python3 .github/ci/data_paths_gate.py --drill + + - name: Data paths gate (the mod's own data is built from its own folder; ExeDir and the null-module path stay in their files) + run: python3 .github/ci/data_paths_gate.py + + - name: Plural vectors gate (the l10n selftest's expected forms are the independent evaluator's) + run: python3 .github/ci/l10n_vectors.py --check + + - name: ImGui submodule (the template gate reads the pinned imgui.h for the calls that take a label) + # Last of the gates' inputs, so no earlier gate reads a submodule it does not read elsewhere. Shallow, + # with a full fetch when a direct fetch of the pinned commit is refused. + run: | + git submodule update --init --depth 1 src/votv-coop/third_party/imgui || git submodule update --init src/votv-coop/third_party/imgui + + - name: Translation template gate (the template is current; only allowlisted files extract; call sites follow the rules) + run: python3 .github/ci/l10n_pot.py --check + + - name: Server tests (the master and the relay) + run: cargo test --locked --manifest-path server/Cargo.toml diff --git a/.gitignore b/.gitignore index 2e04e310c..df7a7c850 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ # ===== Game install — never commit copyrighted binaries or assets ===== -Game/ +# Anchored to the root, where the installs live: unanchored, Game_*/ also took src/.../coop/game_build/ on a +# case-insensitive checkout. +/Game/ # Our own toolset. Untracked whole: it is the workflow that maintains this project, not # something a person forking it runs. What the CI genuinely needs lives in .github/ci. tools/ -Game_*/ +/Game_*/ *.exe *.dll *.pak @@ -241,6 +243,7 @@ docs/THUNDERSTORE.md # free and safe by default. The files that left the tree stay on disk; history is not rewritten. docs/** !docs/architecture.md +!docs/bytecode-fingerprint.md !docs/code-map.md !docs/coop-dispatch-visibility.md !docs/coop-entity-expression-map.md @@ -257,6 +260,7 @@ docs/** !docs/status.md !docs/votv-ui-style.md !docs/devices.md +!docs/base-computer.md !docs/deployables.md !docs/events-and-weather.md !docs/join.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19759893c..cebb98354 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,8 @@ A change is not done when it compiles. Before a pull request: Every push and pull request runs the source gates (`.github/workflows/repo-gates.yml`): the config-registry gate, the peer-slot generation gate, the atlas gate, the master-contact and -reliable-kind gates, the MinHook free-call and GC-pin rules, the public-leak gate, and the +reliable-kind gates, the MinHook free-call and GC-pin rules, the workflow-expression gate (a +workflow script reads its values through `env:`), the public-leak gate, and the commit-message and public-prose checks described below. The Windows build is run by hand (Actions -> **build** -> Run workflow); it is what runs the zero-import ABI gate on the built DLL and the package drill, since those need a compiled artifact. diff --git a/README.md b/README.md index dd8e4dcfa..052f3afcd 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ with its own signatures, drives the game's own classes and functions through ref where reflection cannot see. No asset edits, no repacked paks. Transport is GameNetworkingSockets: 13 unreliable pose and state streams, a voice stream beside -them, and one reliable ordered channel whose payloads are 161 message kinds. Each machine's engine +them, and one reliable ordered channel whose payloads are 182 message kinds. Each machine's engine re-derives animation, physics and rendering from the streamed state. The host is authoritative for world state, randomness and NPC simulation; a client acts by naming an intent that the host performs. diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index fbfca4eed..ecc68a050 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -31,7 +31,8 @@ Portions of this software are copyright (c) 2006-2024 The FreeType Project Statically linked libraries: MinHook, Dear ImGui, GameNetworkingSockets (bundling Abseil, WebRTC, curve25519-donna, ed25519-donna), Opus, FreeType, -miniaudio, miniz, Protocol Buffers, OpenSSL, nlohmann/json. Ported code: RE-UE4SS, LuckPerms. +miniaudio, miniz, Protocol Buffers, OpenSSL, nlohmann/json. Ported code: RE-UE4SS, LuckPerms, +UAssetAPI. Embedded fonts: Roboto, JetBrains Mono, Cascadia Code, Fixedsys Excelsior, Twemoji Mozilla. @@ -1075,6 +1076,34 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## UAssetAPI (ported algorithm) + +No UAssetAPI code is linked; the operand layout of each UE4.27 Kismet bytecode instruction, as UAssetAPI's expression readers read it with their in-memory (iCode) accounting, was PORTED into the opcode table of `src/votv-coop/src/ue_wrap/core/script_bytecode.cpp` with an attribution comment, and that portion carries the upstream MIT notice. Upstream: https://github.com/atenfyr/UAssetAPI (read from the fork https://github.com/Yangff/UAssetAPI). Source of this text: that fork's `LICENSE` at commit `0b4d2cff87bd391d56e9a208e35c92d77238bacb`, read from a local clone (the tree itself is not carried by this repository). + +```text +MIT License + +Copyright (c) 2023 Atenfyr + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + --- The fonts below are embedded in `main.dll` as PE resources and merged into the diff --git a/docs/README.md b/docs/README.md index f15a8306e..717964b56 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,6 +36,7 @@ Then, if you are going to touch code: | [testing.md](testing.md) | The test rig, the launcher and its scenarios, the in-game harness, and the gates a push runs | | [release.md](release.md) | How a build becomes a release, and the gates it must pass | | [versioning.md](versioning.md) | The version pair two peers must match, the files a game update touches, how a broken build says so, and the port runbook | +| [bytecode-fingerprint.md](bytecode-fingerprint.md) | How the mod tells that a game function it depends on changed its logic: the canonical text of Blueprint bytecode and its fingerprints | | [votv-ui-style.md](votv-ui-style.md) | The game's own widget style, measured; binding for anything drawn inside the game's UI | Before writing any entity-sync, hook or spawn-catch code, read these three: @@ -61,7 +62,8 @@ messages, the late-join answer, the known limits with their evidence, and the co | [npcs-and-kerfur.md](npcs-and-kerfur.md) | Host-owned characters, the creatures each peer owns, the killer wisp, the roaches, and the kerfur robot across its two forms | | [events-and-weather.md](events-and-weather.md) | The clock, the weather, who rolls, the story events and how a client sees one fire, the event actors, the pyramid, the alarm, emails and the task | | [signals.md](signals.md) | The signal workstation: occupancy, the desk, the download simulation, the catch and the dishes, the deck, drives and racks, the laptop, the database, and the upgrades | -| [devices.md](devices.md) | Doors, keypads, lights, the power panel, the turbine, windows and grime, the drone, and the economy: the balance, shop orders, the coin gun | +| [devices.md](devices.md) | Doors, keypads, lights, the power panel, the turbine, windows and grime, coffee appliances, the drone, the signal servers | +| [base-computer.md](base-computer.md) | The base laptop and the economy: the balance, upgrades, the laptop's tab and inbox, shop orders, the coin gun, the floppy slot | | [vehicles.md](vehicles.md) | The ATV: a mirror that runs the vehicle's physics natively and is corrected, the two authority predicates, the condition that travels | | [ui.md](ui.md) | The native screens, the F1 overlay, the HUD, and the game's notifications | | [send-path.md](send-path.md) | How fast each connection is paced, measured rather than configured; who may fill a send buffer, and where a held-back send waits | diff --git a/docs/architecture.md b/docs/architecture.md index 9f3da3311..565a5a118 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -127,8 +127,9 @@ refuses, and the packet header is the backstop. ## Identity -**Peers.** Every install holds an Ed25519 keypair in a key file beside the game executable, -under a private access list (`coop/net/peer_identity`); the public key *is* +**Peers.** Every install holds an Ed25519 keypair in a key file in the mod's own folder, +bound to the machine it was made on and under a private access list +(`coop/net/peer_identity`); the public key *is* the peer's network identity, and the id the master server and the per-player inventory store use is derived from it. A name proves nothing by itself, so before a host spends a seat the two ends sign each other's nonces with the key their identity names @@ -192,7 +193,7 @@ a call is a property of how the Blueprint VM dispatches it, not of the function: |---|---|---| | the `ProcessEvent` interceptor | engine-originated calls: ticks, BeginPlay, input events, delegates, timers, interface events | yes | | the script-body gate (`ue_wrap/core/script_gate`) | every Blueprint function body, on every route (an engine event, a Blueprint's own local call, a call through a context switch, an ubergraph entry; a virtual or final call to a script function ends in the same loop), with its arguments and the frame that called it | yes, per call | -| the native function seam (`ue_wrap/core/ufunction_hook`) | calls into native functions on every route, including from inside a Blueprint's own graph | before the call, for a native with no return value and no out or reference parameter (its PRE form); after it, no | +| the native function seam (`ue_wrap/core/ufunction_hook`) | calls into native functions on every route, including from inside a Blueprint's own graph | before the call, for a native with no return value and no out parameter or non-const reference (its PRE form); after it, no | | per-site reconcile | anything else: let the verb run, diff the observable state, converge to the authority's answer | after the fact | [coop-dispatch-visibility.md](coop-dispatch-visibility.md) is the per-function table and the diff --git a/docs/base-computer.md b/docs/base-computer.md new file mode 100644 index 000000000..2052a3adf --- /dev/null +++ b/docs/base-computer.md @@ -0,0 +1,237 @@ +# The base computer and the station's economy + +## Purpose + +The laptop and the money the station runs on: the shared balance, the upgrade levels, the laptop's +tab, its inbox, the shop orders and the delivery queue, the coin gun, and the floppy slot of a +laptop or a signal server. Who owns each, how a client's purchase or order reaches the host, and +where the group's money can still be lost. These ride the replication engine and the host-owned +drone described in [devices.md](devices.md). + +## How it works + +### The balance + +The host owns the balance. It polls the points field every tick, which catches every writer, and +broadcasts the absolute value on change and to a joiner; a client writes the host's value directly. +The wire is one-way. A client-to-host balance delta once existed with no bound, so any peer could +set the group's money to anything; it was retired whole rather than clamped +(`coop/world/balance_sync`). A client's own earnings are therefore not shared unless a lane +carries them as an intent. + +### The upgrades + +The eighteen upgrade levels are one struct on the save, so they used to ride only the transferred +save and diverged in silence from the moment a level was bought. The host now polls the struct and +broadcasts it whole on any change, which catches every writer: the panel, the physical racks, the +transformer prop. A client applies what it is sent and repaints the rows it has open. + +A purchase is an intent. A client's button is refused at the script-body gate, so it never debits +itself, and the host re-derives the price and the bounds from its own table and its own level, +charges the shared balance and republishes. The rows have no identity of their own to name, so +holding the laptop claim is the sender's reach, and the index must be one of the fifteen rows that +buy a level. The host asks exactly what the button asks, including the game's own asymmetry: the +balance is tested against the row's unaccumulated price and charged the accumulated one. + +### The laptop's tab + +The base laptop and every portable PC show one widget whose tabs are a switcher. The peer holding the +laptop claim reports its index after each tab change (`coop/interactables/laptop_tab`); the host checks +that the sender holds the claim and the index is one of its own tabs, writes it to its widget and sends +it on. A peer that does not hold the claim writes the switcher's index and nothing else: the game's own +tab call also binds that peer's keys to a remote drone. A joiner gets the host's index at its world-ready. + +### The laptop's inbox + +The laptop's messages -- the hash-collection task mails, the scientist and alien replies, the +mails an event or a caught signal produces -- are the same on every machine, and deleting one +deletes it for everyone. Every producer in the game funnels through one Blueprint-internal +function into the save's email array, and calling that same function on a receiver reproduces the +whole arrival at once: the stored row, the list entry, the ding at the physical laptop and the +tab highlight, with the date re-stamped from the synced clock. + +Each peer keeps a shadow of the array and diffs it once a second. The array only ever grows at +the tail, so the diff is positional. A new row is broadcast by the host -- see below -- and a +removed row is broadcast by whoever deleted it, as the row's content HASH and never its index, +because a producer writes its own row before that row reaches anyone and two peers therefore hold +the same messages in different orders. + +Writing a mail is the host's alone: nothing in the game authors a mail from a player action, so a +client that started producing them could only be a diverged simulation writing into everyone's +permanent inbox, and a client's append is dropped at the host rather than relayed. Deleting is +symmetric, because the only thing in the game that removes a mail is a player pressing the row's +delete button (`coop/world/email_sync`). + +### Shop orders + +A client's laptop order is entirely local to its machine: the game's order function queues it in +the client's own save and hands it to the client's own mirror drone. So on a client the order function's +queueing (`addOrderCart`) and its send to the drone (`sendShop`) are refused at the script-body gate, and +the order goes to the host as an intent naming each item's shop row and nothing else, read from the +function's own parameter at its entry; the host prices the row from its own +store table, checks its own balance, rolls its own delivery time, commits through the game's own +order function and charges (`coop/items/order_sync`). An order a world event makes on a client (the +daily delivery, a gift) is not sent: the host's copy of the event makes it. A refused order comes back with a reason, and the refused items +are put back in the client's cart, because the game's own affordability gate runs before the cart +is cleared while the refusal arrives after. This is the reference intent lane: the intent used to +carry a client-chosen price, and every client shopped free. + +The intent names a shop ROW, not an object class, because a class cannot name an item: the game's +473 shop rows map onto 368 distinct classes, and `prop_C` alone is shared by 50 of them, so 112 +rows have no unique class. The row name is the shop's real identity, and the game stamps it into +each generated store entry, so a forwarded order already carries it. + +The host's price comes from the game's own `list_store` table, read two independent ways -- a walk +over the row map, and a fully reflected column read that needs no layout knowledge. They are +compared, and a single disagreement invalidates the whole catalog: the host then refuses client +orders outright rather than charge a number it cannot vouch for. Setting +`VOTVCOOP_STORE_CATALOG_BREAK=1` makes the walk read the wrong field on purpose, so that refusal +can be seen firing before it is trusted (`ue_wrap/world/store_catalog`). + +The delivery queue is the host's. Each change to it -- an order queued, the delivered one taken off -- +goes to every client as the laptop's own queue functions run, and a client's queue follows through +the same functions (`coop/items/order_queue_sync`). A shop item travels by its row; an item a world +event built outside the shop has no row and travels by its class, and by the list_props name a generic +prop carries in its asProp (the daily delivery's reel case), rebuilt in the one shape every such builder +makes. A change waits, in its place, while the client's laptop, its shop catalog or the engine's name +conversion is not ready yet; an item the client can never build is left out, down to an order of no +items that still keeps its place, so the host's next delivery takes the same order off both queues; and +a change counts as applied only when the queue moved by exactly one. + +### The coin gun + +Shooting a prop with the coin gun sells it: the game destroys the prop and mints coins whose +material is their denomination. A client's shot is uncancellable, so the lane captures the +client's own coins at their birth and releases or destroys them at the next barrier, sends a sale +intent naming the prop by key ahead of the client's ordinary destroy on the same lane, and the +host resolves the prop in its own world, prices it from its own copy, mints through the gun's own +sell function and destroys the sold prop itself; the coins are host-owned world actors that the +event-actor mirror carries, and whoever's body trips one on the host credits the host, a client's +puppet included (`coop/items/coingun_sync`). The client is told the result. + +### The floppy slot + +A laptop and a signal server hold a disc the same way: inserting one moves its type, its remaining +writes, its data rows and the JSON of its whole save struct into four fields of the device and +destroys the actor, and ejecting one spawns the disc back from those fields. The four are one +concept with one owner, and they are state, not an event: the host holds every device's slot, each +peer polls its own devices once a second behind a digest that reads the raw field bytes, and a peer +whose own game changed a slot sends the host the outcome as a claim. The host applies it and answers +with the canonical, which is also the acknowledgement (`coop/interactables/floppy_slot_sync`). + +The laptop also edits the files of the disc it holds, which its file quad carries as edit scripts +(`coop/interactables/laptop_buffer_sync`). So the laptop's slot is watched for which disc it holds +alone: the rows and the writes travel with a change of that and are the quad's between two. A +canonical naming the disc a laptop already holds leaves its files alone, and the slot lane primes +the quad at each change it takes -- a claim sent, a publish, a canonical written -- so an edit made +after it is the quad's to send, even one made before the host answered the claim. The host counts +each change of the laptop's disc; the quad's batches and canonicals carry that count on the slot's +lane, a batch made on another disc is answered with the canonical instead of applied, a canonical of +another disc is dropped, and neither end applies or sends one while its own change of the disc waits +for the other. + +Emptying a slot writes only what the device's own eject writes -- the type and the rows. The eject +runs in two phases: it clears those two, and about a second later, when the carrier's timeline +finishes, the deferred spawn reads the remaining writes and the save JSON to rebuild the disc. +Anything that zeroes those in between hands the player back a blank disc under a new identity. + +A disc a peer ejects reaches the others through the ordinary birth channel: a client's own fresh +prop spawn is not broadcast, so its device's eject is reported to the host, which authors the disc +and broadcasts it like any other world prop. Three other lineages travel that way, and all three +are born into a hand and spawn inert on the host until the holder's pose stream drives them; a +disc is not held by anyone, so it falls on the host instead. Either way the disc's files and +remaining writes travel as its save record behind the birth; a disc the host ejects gets its record +from the host's finish-spawning drain, since the eject fills the disc in after the finish call +([props.md](props.md)). + +The laptop's slot crosses as one blob, and the save JSON carries the rows a second time, so a disc +with a few dozen rows is kilobytes. The blob is held to the ceiling the server slot uses and refused +whole past it, never cut: a cut JSON is one the receiver's own eject cannot parse, and the disc it +hands back comes out blank under a new key. The laptop keeps at most 31 rows in the slot itself +(`ui_laptop.updFloppy` trims `floppyData`); the JSON keeps all of them. + +A device takes a disc by two entries, and only one of them is a player. Pressing E with a disc in +hand is deliberate, happens on one machine, and the slot carries its outcome. The other is the +hitbox reporting whatever touches the slot -- and an eject spawns the disc INSIDE the box it came +out of, which the Blueprint handles by turning that device's own hitbox off for a moment. In single player +the only birth that can land in a slot is that eject, so guarding the one box is enough; in coop +the disc is also born on the other machine, in a box whose hitbox nobody turned off and where +nobody ejected anything. So the rule sits on the disc rather than the box: a disc is in transit for +a moment after it materialises, and no device swallows one in transit, on any peer. The window is +anchored at the disc's own appearance on each machine, so both peers run one rule against one local +event and neither waits on a message; when it lapses, the native rule simply resumes. + +The window is one number for every device, because the mark is on the disc and a disc does not know +which slot it came out of. It has to clear the frame or two the hitbox entry needs to fire, and it +has to end before the device that ejected re-enables its own hitbox -- re-enabling a collider +re-reports every body already inside it, which is how the game re-takes a disc still sitting in the +slot, and that report comes once. The two devices disagree on the pause: a signal server waits a +second, a laptop half of one. The window is set under the shorter of the two, so neither device's +own re-take is touched and an ejecting peer behaves exactly as it does in single player. + +The hitbox also takes a disc another peer is holding. That peer's pose drives this machine's copy +of the disc, and bringing it to the slot brings the copy into this machine's copy of the slot, so +the watching peer would insert it too and claim a second insert of one disc. The entry refuses a +disc under another peer's hold on every peer; the holder's own insert is the one the slot carries. + +## Who owns what + +| State | Owner | Shape | +|---|---|---| +| the balance | the host | one-way, absolute, on change | +| the upgrade levels | the host | the struct whole, on change; a purchase is an intent | +| an order | the client names the row; the host performs and prices | an intent | +| a coin gun sale | the client names the prop; the host prices, mints and destroys | an intent ahead of the destroy | +| a device's floppy slot | the host | a 1 Hz digest-gated poll; a peer claims the outcome of its own insert or eject, and the host's canonical is the answer | + +## Wire messages + +| Kind | Direction | Carries | +|---|---|---| +| `LaptopTabReport`, `LaptopTabState` | the claim holder to the host; the host to the other clients | the switcher's index | +| `BalanceSync` | the host to all | the absolute balance | +| `UpgradeLevels` | the host to all | the eighteen levels, whole | +| `UpgradeIntent` | a client to the host | the panel row's index and whether it is a buy or a sell | +| `OrderRequest`, `OrderRefused` | a client to the host; the host to one client | the items by row; a refusal and its reason | +| `OrderQueue` | the host to all | a change to the delivery queue: a reset, an order queued (its items by row or by class), the first one taken off | +| `CoinGunSell`, `CoinGunResult`, `CoinCollect` | a client to the host; the host to one client; a client to the host | the sold prop's key; the outcome; a coin the client tripped | +| `FloppySlotState` | a peer to the host with a claim; the host to all with the canonical | one device's slot, or a set of them -- the server boxes' and the laptop's: the type, the writes, the rows and the save JSON, and the laptop's generation | + +## Late join + +The balance is sent to a joiner at connect. The delivery queue goes to a +joiner as a reset and then every queued order, and the joiner ignores the queue's changes until that +reset arrives. + +Every device's floppy slot goes to a joiner at the same edge, an empty one as much as a full one: +the joiner's world came from the host's save file, which coop stops the game refreshing, so it knows +nothing the host has done since. The laptop's file quad follows the laptop's slot on the same lane +and under the same count, so the joiner takes the disc before its files, and a set the transport +refused is sent again with the quad behind it. + +The inbox rides the joiner's save transfer, so it arrives whole. A mail written during the 30 to +60 seconds the joiner spends loading is in neither that save nor any later diff, so the host +captures the array at the instant it hands over the save and sends the difference on the joiner's +ready edge -- once per slot. A mail that arrives before the joiner's world can take it is parked +in arrival order and applied once the array settles, rather than dropped on the spot; a row the +settled world still refuses after thirty attempts is malformed for that world and is dropped with +an error line. + +## Known limits + +| Limit | Evidence | +|---|---| +| A refused coin-gun sale has already destroyed the prop on the client, and no heal re-asserts it: a client authors that destruction before the arbiter answers, as its generator install does, whose refusal refunds the upgrade. The host rebuilds its key index periodically, so a refusal is rare | `[V]` `coop/items/coingun_sync` | +| The coin collect has two entries; the interceptor sits on the overlap entry, and the E-press entry dispatches inside the Blueprint where it cannot fire, so a coin a client collects by pressing is credited on the client only and the host's next balance broadcast erases it | `[V]` `coop/items/coingun_sync` | +| A client's earnings from a chest or an achievement reach only its own machine and are erased by the host's next broadcast | `[V]` `coop/world/balance_sync` is one-way | +| A slot change reaches the other peer on the next poll, so up to a second plus the round trip. A player who reaches a box inside that window acts on the slot as it was: an eject of a disc the other peer has just inserted answers "No floppy disc in the slot" and is not retried, and a client's insert into a slot the host has just filled replaces the host's disc when the claim lands, which loses that disc | `[V]` the lane polls at 1 Hz, and the host applies a claim over whatever its slot holds (`coop/interactables/floppy_slot_sync`); a faster poll would narrow the window rather than close it | + +## Code map + +| Concept | Files | +|---|---| +| the laptop's tab | `coop/interactables/laptop_tab` | +| the floppy slot | `coop/interactables/floppy_slot_sync`, `coop/interactables/laptop_buffer_sync`, `ue_wrap/devices/floppy_slot`, `ue_wrap/devices/serverbox`, `ue_wrap/devices/laptop` | +| the inbox | `coop/world/email_sync`, `ue_wrap/world/email`, `coop/session/join_seed` | +| the economy | `coop/world/balance_sync`, `coop/items/order_sync`, `coop/items/order_queue_sync`, `coop/items/coingun_sync`, `coop/interactables/upgrade_sync`, `ue_wrap/world/economy`, `ue_wrap/world/order_economy`, `ue_wrap/world/store_catalog`, `ue_wrap/world/upgrades` | +| tests and probes | `coop/dev/order_selftest`, `coop/dev/floppy_selftest` | diff --git a/docs/bytecode-fingerprint.md b/docs/bytecode-fingerprint.md new file mode 100644 index 000000000..6eead586c --- /dev/null +++ b/docs/bytecode-fingerprint.md @@ -0,0 +1,225 @@ +# Bytecode fingerprints + +## Purpose + +A game update can keep every class and function name the mod uses and still change what a function +does. Resolving names by string cannot see that. A fingerprint can: the mod renders a game function's +compiled Blueprint bytecode as canonical text and hashes it, so the same function on two builds of the +game compares equal when its logic is the same and differs when it is not. This page is the +specification of that text. The mod and the port tooling each render it, one from the running game and +one from the cooked assets, and the two renderings are compared function by function. + +## How it works + +### Reading the bytecode + +A Blueprint function's script is the byte array `UStruct::Script`. The mod walks it expression by +expression with a decoder of every UE4.27 Kismet opcode, laid out as the engine keeps it in memory: an +object or property pointer takes 8 bytes and a name (`FScriptName`) 12. The operand order of each +opcode, and its name, follow UAssetAPI's expression readers (MIT): an expression is written with the +name of its `EExprToken` less `EX_`. A decoder that mis-sized one operand would not land on the script's +end; on both game builds it has read every function of every loaded Blueprint class to its end with no +problem `[V]` (every function of a capture: 21,087 on 0.9.0n, 22,762 on 0.10.0, in a world). Expressions nested more than +256 deep are refused as a corrupt script; a compiled Blueprint nests a few dozen deep. + +### The canonical text + +1. **Line 1 is the signature**: `sig(` then the function's parameters (the properties flagged + `CPF_Parm`) in declaration order, comma-separated, each `name:flags:type`, then `)` and a newline. + The flags are letters in the order C (const), P (parameter), O (out), R (return), F (reference). The + type is the name of the property's class (`objectproperty`, `intproperty`, ...), followed by: + - `:` of the property's type object, when it has one, for an object, soft object, weak + object, lazy object, class or soft class property (the class it holds: `class` for a class + property), an interface property (its interface), a struct property (its struct), a byte property + (its enum, if it has one), and a delegate or any multicast delegate property (its signature + function); + - for an enum property, `:`; + - for an array or set property ``, for a map property ``, each + written by the same rule, to a depth of four (deeper, the class name alone). + There is no size: a native struct's cooked size is not its run-time one (`FHitResult` reads 144 bytes + in the asset and 136 in the game `[V]`), and the type already names the struct. A native function has + no script, so its text is this line alone. +2. **Statements in flow order**, one per line as `