From 706a50787f0dfd977005d132b3af1afab451bc92 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 7 Sep 2026 18:24:23 -0400 Subject: [PATCH 1/2] ci: qualify post-v0.4.0 correctness candidate --- .github/workflows/validate_wrapper.yml | 62 ++++++++++++++++++++------ AGENTS.md | 2 + docs/post_v040_qualification.md | 32 +++++++++++++ tools/validate_grammar_boundary.py | 43 ++++++++++++++++++ 4 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 docs/post_v040_qualification.md create mode 100644 tools/validate_grammar_boundary.py diff --git a/.github/workflows/validate_wrapper.yml b/.github/workflows/validate_wrapper.yml index 54048e9..9b9e9d9 100644 --- a/.github/workflows/validate_wrapper.yml +++ b/.github/workflows/validate_wrapper.yml @@ -16,6 +16,7 @@ on: - 'tools/package_linux_artifact.py' - 'tools/validate_linux_artifact.py' - 'tools/validate_exports.py' + - 'tools/validate_grammar_boundary.py' - 'third_party/llama.cpp' push: branches: @@ -34,11 +35,16 @@ on: - 'tools/package_linux_artifact.py' - 'tools/validate_linux_artifact.py' - 'tools/validate_exports.py' + - 'tools/validate_grammar_boundary.py' - 'third_party/llama.cpp' permissions: contents: read +env: + # Qualification only: never moves the production submodule or release channel. + LLAMADART_QUALIFICATION_SHA: 73ab7599b553c03f6f5d2db24a18ad76f2eb36a3 + jobs: android-arm64-isa: runs-on: ubuntu-latest @@ -50,8 +56,9 @@ jobs: submodules: recursive - name: Select candidate upstream run: | - git -C third_party/llama.cpp fetch --depth=1 origin tag v0.4.0 - git -C third_party/llama.cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - uses: actions/setup-java@v5 with: distribution: temurin @@ -91,8 +98,9 @@ jobs: submodules: recursive - name: Select candidate upstream run: | - git -C third_party/llama.cpp fetch --depth=1 origin tag v0.4.0 - git -C third_party/llama.cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - name: Install cross compiler and CPU emulator run: sudo apt-get update && sudo apt-get install -y ninja-build gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user - name: Build the real baseline CPU and dispatch test @@ -121,7 +129,7 @@ jobs: strategy: fail-fast: false matrix: - upstream: [pinned, v0.4.0] + upstream: [pinned, post-v0.4.0] steps: - uses: actions/checkout@v7 with: @@ -132,11 +140,12 @@ jobs: with: arch: arm64 - name: Select candidate upstream - if: matrix.upstream == 'v0.4.0' + if: matrix.upstream == 'post-v0.4.0' shell: bash run: | - git -C third_party/llama.cpp fetch --depth=1 origin tag v0.4.0 - git -C third_party/llama.cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - name: Verify assembly policy regressions run: python -m unittest discover -s tests -p test_kleidiai_windows.py - name: Configure release compiler and optimized CPU @@ -155,16 +164,25 @@ jobs: ctest --test-dir build/wa64 -C Release --timeout 120 --verbose --output-on-failure linux-artifact-contract: + name: ${{ matrix.upstream == 'pinned' && format('linux-artifact-contract ({0})', matrix.arch) || format('linux-artifact-contract ({0}, post-v0.4.0)', matrix.arch) }} runs-on: ubuntu-latest + timeout-minutes: 30 strategy: fail-fast: false matrix: arch: [x64, arm64] + upstream: [pinned, post-v0.4.0] steps: - uses: actions/checkout@v7 with: persist-credentials: false submodules: recursive + - name: Select candidate upstream + if: matrix.upstream == 'post-v0.4.0' + run: | + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - name: Install Linux build and emulation dependencies run: | sudo apt-get update @@ -196,22 +214,24 @@ jobs: fi wrapper-contract: - name: ${{ matrix.upstream == 'pinned' && 'wrapper-contract' || 'wrapper-contract (v0.4.0)' }} + name: ${{ matrix.upstream == 'pinned' && 'wrapper-contract' || 'wrapper-contract (post-v0.4.0)' }} runs-on: ubuntu-latest + timeout-minutes: 30 strategy: fail-fast: false matrix: - upstream: [pinned, v0.4.0] + upstream: [pinned, post-v0.4.0] steps: - uses: actions/checkout@v7 with: submodules: recursive persist-credentials: false - name: Select the candidate upstream release - if: matrix.upstream == 'v0.4.0' + if: matrix.upstream == 'post-v0.4.0' run: | - git -C third_party/llama.cpp fetch --depth=1 origin tag v0.4.0 - git -C third_party/llama.cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - name: Configure wrapper contract tests run: >- cmake -S . -B build/wrapper-contract -G Ninja @@ -229,6 +249,9 @@ jobs: llamadart_tts_smoke - name: Run wrapper contract tests run: ctest --test-dir build/wrapper-contract --output-on-failure + - name: Validate candidate public grammar sampler boundary + if: matrix.upstream == 'post-v0.4.0' + run: python3 tools/validate_grammar_boundary.py build/wrapper-contract/libllamadart.so - name: Validate wrapper exports run: >- python3 tools/validate_exports.py @@ -237,12 +260,25 @@ jobs: build/wrapper-contract/libllamadart.so msvc-mtmd-link-contract: + name: ${{ matrix.upstream == 'pinned' && 'msvc-mtmd-link-contract' || 'msvc-mtmd-link-contract (post-v0.4.0)' }} runs-on: windows-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + upstream: [pinned, post-v0.4.0] steps: - uses: actions/checkout@v7 with: submodules: recursive + persist-credentials: false + - name: Select candidate upstream + if: matrix.upstream == 'post-v0.4.0' + shell: bash + run: | + git -C third_party/llama.cpp fetch --depth=1 origin "$LLAMADART_QUALIFICATION_SHA" + git -C third_party/llama.cpp checkout --detach "$LLAMADART_QUALIFICATION_SHA" + test "$(git -C third_party/llama.cpp rev-parse HEAD)" = "$LLAMADART_QUALIFICATION_SHA" - name: Setup MSVC uses: TheMrMilchmann/setup-msvc-dev@v4.1.0 with: diff --git a/AGENTS.md b/AGENTS.md index b281e11..22be65f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,8 @@ For ARM64 upstream upgrades, run the candidate Windows/Android and compiled dispatch gates in `validate_wrapper.yml` before merging. See `docs/platform_backend_strategy.md` for the ISA audit contract. Never refresh audited source fingerprints merely to silence a failed check. +See `docs/post_v040_qualification.md` for the exact nonpublishing correctness +candidate, its separate pinned lanes, and remaining artifact/hardware gates. - `.github/workflows/native_release.yml` - Exact native build + release workflow, used by manual dispatch and the diff --git a/docs/post_v040_qualification.md b/docs/post_v040_qualification.md new file mode 100644 index 0000000..1d2d9f3 --- /dev/null +++ b/docs/post_v040_qualification.md @@ -0,0 +1,32 @@ +# Post-v0.4.0 correctness qualification + +Issue #76 qualifies exact upstream commit +`73ab7599b553c03f6f5d2db24a18ad76f2eb36a3`, separately from the production +submodule and published v0.4.0. It contains the grammar repetition threshold +fix (#28469), GDN normalization (#28068), Metal early-return leak (#28399), +and CUDA race/barrier fixes (#28475/#27870). + +`validate_wrapper.yml` retains production-pinned Windows ARM64, Linux wrapper, +Linux x64/arm64 artifact, and Windows x64 link checks. Its explicitly named +post-v0.4.0 counterparts fetch and verify the immutable candidate SHA. +Android ISA and emulated KleidiAI qualification also target this candidate. +The jobs have read-only repository permissions and do not dispatch, publish, +tag, ingest attestations, or change any release pin. Candidate failures block +candidate readiness, not an assertion that currently pinned artifacts regressed. + +The candidate Linux wrapper additionally calls the built library's public +grammar sampler through `tools/validate_grammar_boundary.py`: 1999 and 2000 +repetitions must initialize, 2001 and malformed syntax must fail. The unchanged +v0.4.0 library fails this test at 2000, so the check runs only on the candidate. +Character-only initialization requires no model vocabulary; this is sampler +initialization evidence, not model inference or token acceptance coverage. + +Local macOS CPU/Metal wrapper tests and exports, a bounded Qwen3.5 model A/B, +and matching bridge state/image/ASR/TTS smokes are recorded in issue #76. +They do not replace the hosted owner-platform checks or the hosted exact-artifact +Web qualification required for publication. CUDA hardware and older Android +remain unavailable/unverified; emulation is not a physical-device result. + +This qualification does not invent a release identity: later upstream source +must not be called a v0.4.0 wrapper rebuild. Adoption requires a containing +upstream stable release or a separately authorized development-channel decision. diff --git a/tools/validate_grammar_boundary.py b/tools/validate_grammar_boundary.py new file mode 100644 index 0000000..0647b3c --- /dev/null +++ b/tools/validate_grammar_boundary.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Exercise the candidate runtime's public grammar sampler at its size limit. + +Qualification only: the production v0.4.0 pin is known to fail the exact 2000 +boundary. No model is needed because these grammars contain only characters; +the vocabulary is not accessed until token sampling. +""" + +import argparse +import ctypes +from pathlib import Path + + +def validate(library: Path) -> None: + runtime = ctypes.CDLL(str(library.resolve())) + initialize = runtime.llama_sampler_init_grammar + initialize.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p] + initialize.restype = ctypes.c_void_p + free = runtime.llama_sampler_free + free.argtypes = [ctypes.c_void_p] + free.restype = None + for grammar, expected in ( + (b'root ::= "a"{1999}', True), + (b'root ::= "a"{2000}', True), + (b'root ::= "a"{2001}', False), + (b'root ::= "a"{', False), + ): + sampler = initialize(None, grammar, b"root") + accepted = bool(sampler) + if sampler: + free(sampler) + if accepted != expected: + raise RuntimeError( + f"grammar boundary mismatch: {grammar!r}: " + f"accepted={accepted}, expected={expected}" + ) + print("Validated grammar sampler acceptance/rejection at repetition 2000") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("library", type=Path) + validate(parser.parse_args().library) From 2d3bdce4cc64cf97bf9f4b0358c6f631adf8a66f Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 7 Sep 2026 18:49:19 -0400 Subject: [PATCH 2/2] test: admit audited candidate ISA source pair --- docs/platform_backend_strategy.md | 2 +- docs/post_v040_qualification.md | 21 +++++++++++++++++ tests/test_android_cpu_isa.py | 39 +++++++++++++++++++++++++++++++ tools/validate_android_cpu_isa.py | 29 +++++++++++++++-------- 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/docs/platform_backend_strategy.md b/docs/platform_backend_strategy.md index 2245422..4047e42 100644 --- a/docs/platform_backend_strategy.md +++ b/docs/platform_backend_strategy.md @@ -41,7 +41,7 @@ ## ARM64 upgrade qualification -`Validate Wrapper` checks pinned and candidate v0.4.0 Windows ARM64 builds, +`Validate Wrapper` checks pinned and exact post-v0.4.0 Windows ARM64 builds, the actual Android ARMv8.2 artifact, and compiled Kleidi selectors plus quantized matrix computation under non-SVE QEMU profiles. QEMU is deterministic CPU compatibility evidence, not physical-device or GPU-performance evidence. diff --git a/docs/post_v040_qualification.md b/docs/post_v040_qualification.md index 1d2d9f3..afd1e50 100644 --- a/docs/post_v040_qualification.md +++ b/docs/post_v040_qualification.md @@ -30,3 +30,24 @@ remain unavailable/unverified; emulation is not a physical-device result. This qualification does not invent a release identity: later upstream source must not be called a v0.4.0 wrapper rebuild. Adoption requires a containing upstream stable release or a separately authorized development-channel decision. + +## Android ISA source audit + +The first candidate build succeeded but correctly failed the pre-existing source +fingerprint gate. A separate audit compared v0.4.0 against the exact candidate: +`ggml/src/ggml-cpu`, CPU feature detection, KleidiAI selectors/tables/callers, +`ggml/include`, and ggml build configuration are byte-unchanged. Of 37 changed +files under `ggml/src`, 36 concern accelerator backends. The shared +`ggml-backend.cpp` delta changes two log levels and removes a capacity-induced +scheduler split; it introduces no optimized-kernel call or feature-mask bypass. +The candidate's compiled selector/quantized-compute test passed under non-SVE +QEMU. Independent Astra source-delta review confirmed this evidence. + +After explicit maintainer approval, the containment policy retains the existing +v0.4.0/KleidiAI pair and adds only candidate ggml SHA-256 +`dcb0f04ebb9654b1fe5ac7cc45737c79e62b116a2063ceda81a7ec1ddb1b20e2` +paired with unchanged Kai SHA-256 +`64189fc613c1c4c3aaeeb6bb12b38d85dd6728cafd2261a5a88f1b77b10fe59c`. +Unknown or cross-combined pairs fail closed. Exact ELF function-range allowlists +and scalable-instruction checks are unchanged. Replacement hosted Android +disassembly remains required; accepting source identity alone is not a pass. diff --git a/tests/test_android_cpu_isa.py b/tests/test_android_cpu_isa.py index 5584603..52cc8e1 100644 --- a/tests/test_android_cpu_isa.py +++ b/tests/test_android_cpu_isa.py @@ -99,6 +99,45 @@ def test_tree_hash_binds_names_contents_and_new_callers(self): (root / "probe.cpp").write_text("guarded") self.assertNotEqual(original, audit.tree_digest(root)) + def test_exact_reviewed_source_pairs_are_required(self): + self.assertEqual(len(audit.SOURCE_PAIRS), 2) + for ggml, kai in audit.SOURCE_PAIRS: + with self.subTest(ggml=ggml): + with mock.patch.object(audit, "tree_digest", side_effect=[ggml, kai]): + audit.validate_sources(Path("llama"), Path("kleidiai")) + for actual in (("changed-ggml", kai), (ggml, "changed-kai")): + with mock.patch.object(audit, "tree_digest", side_effect=actual): + with self.assertRaisesRegex(ValueError, "Unaudited"): + audit.validate_sources(Path("llama"), Path("kleidiai")) + + def test_individually_known_but_unreviewed_pair_is_rejected(self): + with mock.patch.object(audit, "SOURCE_PAIRS", frozenset({("g1", "k1"), ("g2", "k2")})), \ + mock.patch.object(audit, "tree_digest", side_effect=["g1", "k2"]): + with self.assertRaisesRegex(ValueError, "Unaudited"): + audit.validate_sources(Path("llama"), Path("kleidiai")) + + def test_added_caller_or_mutated_kernel_invalidates_approved_pair(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + ggml = base / "llama" / "ggml" / "src" + kai = base / "kleidiai" / "kai" + ggml.mkdir(parents=True) + kai.mkdir(parents=True) + (ggml / "selector.cpp").write_text("guarded") + kernel = kai / "kernel.c" + kernel.write_text("audited") + pairs = frozenset({(audit.tree_digest(ggml), audit.tree_digest(kai))}) + with mock.patch.object(audit, "SOURCE_PAIRS", pairs): + audit.validate_sources(base / "llama", base / "kleidiai") + caller = ggml / "new_caller.cpp" + caller.write_text("unguarded") + with self.assertRaisesRegex(ValueError, "Unaudited"): + audit.validate_sources(base / "llama", base / "kleidiai") + caller.unlink() + kernel.write_text("mutated") + with self.assertRaisesRegex(ValueError, "Unaudited"): + audit.validate_sources(base / "llama", base / "kleidiai") + def test_missing_source_tree_fails_closed(self): with tempfile.TemporaryDirectory() as directory: with self.assertRaisesRegex(ValueError, "empty"): diff --git a/tools/validate_android_cpu_isa.py b/tools/validate_android_cpu_isa.py index f06f02e..de6bb52 100644 --- a/tools/validate_android_cpu_isa.py +++ b/tools/validate_android_cpu_isa.py @@ -17,12 +17,19 @@ import sys -# llama.cpp v0.4.0 / KleidiAI v1.24.0. Bind the complete source subtrees, not -# merely a selector fragment: a new caller can invalidate a dispatch audit. -SOURCE_TREES = { - "ggml/src": "c4dc92a7d95ebfad7f5f55e75be2ae773b7d95faf72a9581c9479c42bc41bca0", - "kai": "64189fc613c1c4c3aaeeb6bb12b38d85dd6728cafd2261a5a88f1b77b10fe59c", -} +# Bind complete source subtree pairs, not independent hash sets: a new caller +# or a different KleidiAI combination invalidates the dispatch audit. +# Candidate audit evidence: docs/post_v040_qualification.md. +SOURCE_PAIRS = frozenset({ + ( # llama.cpp v0.4.0 / KleidiAI v1.24.0 + "c4dc92a7d95ebfad7f5f55e75be2ae773b7d95faf72a9581c9479c42bc41bca0", + "64189fc613c1c4c3aaeeb6bb12b38d85dd6728cafd2261a5a88f1b77b10fe59c", + ), + ( # Exact upstream 73ab7599b553c03f6f5d2db24a18ad76f2eb36a3, same KleidiAI + "dcb0f04ebb9654b1fe5ac7cc45737c79e62b116a2063ceda81a7ec1ddb1b20e2", + "64189fc613c1c4c3aaeeb6bb12b38d85dd6728cafd2261a5a88f1b77b10fe59c", + ), +}) # Exact ELF STT_FUNC ranges; never allow by kai_* prefix or disassembly label. # Assembly kernels have nested local labels, which are NOT function boundaries. @@ -78,10 +85,12 @@ def tree_digest(root): def validate_sources(llama_source, kleidiai_source): - for base, subtree in ((llama_source, "ggml/src"), (kleidiai_source, "kai")): - actual = tree_digest(Path(base) / subtree) - if actual != SOURCE_TREES[subtree]: - raise ValueError(f"Unaudited {subtree} source fingerprint {actual}; review dispatch before updating policy") + actual = (tree_digest(Path(llama_source) / "ggml/src"), + tree_digest(Path(kleidiai_source) / "kai")) + if actual not in SOURCE_PAIRS: + raise ValueError( + f"Unaudited ggml/src + kai source fingerprint pair {actual}; " + "review dispatch before updating policy") def function_ranges(symbol_text):