Audit: all 63 bugs + the improvements (phases 1-4)

.github/workflows/ci.yml

@@ -0,0 +1,104 @@
+# BerryAPI CI (improvement 96).
+#
+# A matrix over the compiler, the sanitizer set and the architecture, plus the
+# static checks. The host suite is the primary gate; the arm64 job runs the
+# portable tests under qemu-user so the arm64 decode paths are exercised on a
+# real arm64 ABI. TSan is deliberately not in the matrix: the core uses atomics
+# and mutexes with a documented thread model, and TSan's false positives on the
+# vendored Luau allocator make it noise rather than signal (see docs/THREADING.md).
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  host:
+    name: host (${{ matrix.cxx }}, ${{ matrix.build }})
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cxx: [g++, clang++]
+        build: [test, sanitize]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install toolchain
+        run: sudo apt-get update && sudo apt-get install -y clang
+      - name: Build and test
+        env:
+          CXX: ${{ matrix.cxx }}
+        run: |
+          if [ "${{ matrix.build }}" = "sanitize" ]; then
+            make -j"$(nproc)" test-sanitize
+          else
+            make -j"$(nproc)" test
+          fi
+
+  arm64:
+    name: arm64 (qemu)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install qemu + cross toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y g++-aarch64-linux-gnu qemu-user-static
+      # The Makefile target cross-compiles both the C++ and the vendored C, so
+      # every object is arm64. (Overriding only CXX here left the C objects
+      # host-format and the link failed with "file in wrong format".)
+      - name: Cross-build the portable suites
+        run: make arm64-portable-tests
+      - name: Run the arm64 decode + parser tests under qemu
+        run: |
+          for b in build/test_test_sigscan build/test_test_x86_length \
+                   build/test_test_url build/test_test_metrics \
+                   build/test_test_inflate build/test_test_netquota \
+                   build/test_test_conn_pool build/test_test_require; do
+            echo "-- $b --"
+            qemu-aarch64-static -L /usr/aarch64-linux-gnu "$b" "$b" || exit 1
+          done
+
+  static-analysis:
+    name: static analysis
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install tools
+        run: sudo apt-get update && sudo apt-get install -y clang-tidy cppcheck
+      - name: clang-tidy (first-party, high-signal checks only)
+        run: |
+          make -j"$(nproc)" compile_commands.json 2>/dev/null || true
+          if [ -f compile_commands.json ]; then
+            clang-tidy -p . --checks='-*,bugprone-*,clang-analyzer-*,performance-*' \
+              core/hooks/*.cpp core/net/net.cpp core/script/*.cpp 2>&1 | tail -50
+          fi
+      - name: cppcheck
+        run: |
+          cppcheck --error-exitcode=1 --enable=warning,performance \
+            --suppress=missingIncludeSystem --inline-suppr \
+            -Iinclude core util 2>&1 | tail -50
+      - name: Strict warning baseline (-Wshadow -Wconversion)
+        run: make check-warnings
+      - name: Script lint/format pack (.luaurc)
+        run: make lint-scripts
+      - name: Script examples run (docs verified in CI)
+        run: make check-examples
+
+  provenance:
+    name: provenance + SBOM
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Vendored dependency pinning
+        run: |
+          make check-mbedtls
+          make check-sources
+          make check-android-sources
+      - name: Generate a CycloneDX SBOM (improvement 100)
+        run: python3 tools/gen-sbom.py > sbom.cdx.json && head -20 sbom.cdx.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sbom
+          path: sbom.cdx.json

.gitignore

@@ -30,3 +30,6 @@ Thumbs.db
 # Python bytecode
 __pycache__/
 *.pyc
+
+# Agent build logs
+.buildlog/

.luaurc

@@ -0,0 +1,22 @@
+{
+  "languageMode": "strict",
+  "lint": {
+    "*": true,
+    "LocalShadow": true,
+    "ImportUnused": true,
+    "UnusedLocal": true,
+    "DuplicateLocal": true,
+    "GlobalShadow": true,
+    "FunctionUnused": true,
+    "TableLiteral": true,
+    "ForRange": true,
+    "UnreachableCode": true
+  },
+  "lintErrors": true,
+  "globals": [
+    "test", "async", "Events", "Http", "network", "ws", "json_rpc",
+    "hud", "ui", "render", "world", "entity", "fs", "audio", "input",
+    "cosmetic", "animator", "perf", "mod", "intent", "crystal", "combat",
+    "packets", "touch", "voice", "gamereads"
+  ]
+}

.openhands/memory/2026-11-04.md

@@ -0,0 +1,73 @@
+# 2026-11-04 — BerryAPI audit fix session
+
+Applied a four-phase subset of the "63 bugs / 100 improvements" audit on branch
+`lynxifyy/berryapi-audit-fixes`, one commit per micro-phase, authored Lynxifyy.
+
+Phase 1 (stability, bugs 1-13): x86 length decoder (0x68/0x69 Iz, Jcc rel32 no
+ModRM, SIB rip-relative, terminal flag); shim prologue stop at terminal, intra-
+span branch refusal, int3-first patch + verify, MAP_FIXED_NOREPLACE address
+check, RW->RX pools with slot free-list, unpatch verifies its own jump, locked
+Backend(); mem::ScopedWritable per-page restore + no RWX fallback.
+
+Phase 2 (security, bugs 30-47, 53-58): HTTP chunked + Content-Length + truncation,
+CRLF/NUL validation, IPv6/userinfo/port authority parse, Host port, CL:0, strict
+status/header parse; WS random key + Sec-WebSocket-Accept verify + Upgrade/
+Connection, CSPRNG mask under lock, carried rx buffer, masked close with code
+validation, strict client frame parser; room token fail-closed, expiry required,
+integer/host validation, CSPRNG self-tag, session key zeroization.
+
+Phase 3 (perf/correctness, bugs 14-29, impr 16-32): signature parser styles +
+half-nibble, maps cache/merge/inode/execute-only, build-id + SHA-256 ASLR-stable
+gate, arm64 adrp/add/xref fixes, memchr anchor, FindUnique/ClosestMatch/
+FindGuarded/FindParallel, SigCache unlock-during-scan, berry-sigcheck CLI.
+
+Phase 4 (DX slice, impr 71, 77, 80): async.all/race/timeout; IDL since/removed +
+breaking-change linter; capabilities.json + manifest.schema.json.
+
+Note: audit "bug 1" (rebase base) was a false positive — the offsets cancel; the
+real bug there was the SIB rip-relative form, which is fixed. Bug 60 (M_PI) does
+not break the host build; left as-is. ASan caught a real over-read in
+ResolveAdrpAdd (bug 29) that the read-through missed; fixed with maxInsns.
+
+## Continued: full audit sweep (same branch, PR #19)
+All 63 bugs addressed. Bugs 48-52 (lifecycle/logging), 59-63 (script hygiene)
+added; bug 63 fixed by removing the unused Berry::String alias.
+
+Improvements implemented+tested: 12,13,14,16,18,20-31,56-58,60-62,66-76,
+77-83,84-92,95-100. NOT implemented (additive features): 17 (full Horspool/SSE2;
+have a memchr anchor), 19 (on-disk offset cache), 32 (exec-only read from disk
+ELF; mapping is flagged only), 33-55 (HTTP proxy/gzip/redirects/pinning/CA-dir/
+happy-eyeballs, WS extras, per-script network quotas), 59 (DAP server).
+
+New test suites: test_metrics, test_crash, test_configfile, test_timers.
+New gates in `make test`: check-sigcheck, check-warnings, check-examples,
+idl-check --golden-test.
+
+Lessons:
+- Do NOT run a build while editing headers: a sanitizer build that links objects
+  from before and after a generated-header change segfaults on a struct-layout
+  mismatch (looks like a real bug). Settle the tree, then build.
+- The Makefile `-include`s `build/tools/*.d`; keep it that way.
+- UBSan flags a deliberate null store in a crash test; use raise(SIGSEGV).
+- ASan caught two real bugs I introduced/missed: Histogram bucket array one
+  short, and the ResolveAdrpAdd over-read.
+
+## Third pass: the rest of the improvements
+Implemented and tested the remaining improvements: hook chains/priorities/
+per-handler counters/auto-disable + ScopedHook/HookGroup (4,8,9,10); single-pass
+multi-pattern scan, Horspool fallback, build-id offset cache, exec-only bytes
+from disk (16,17,19,32); gzip/deflate, HTTPS->HTTP redirect refusal, keep-alive
+pool, streaming responses, cert pinning, CA dir, HTTP CONNECT/SOCKS5, Happy
+Eyeballs, WS fragmentation/keepalive, network quotas (34-45,51,53,55);
+per-binding profiler + Chrome trace, sandboxed require, bytecode cache, consent
+store, secret store, bus payload validation, replay determinism, config hot
+reload, log ring/JSON, secure buffer, metrics debug endpoint (56-76,84-95);
+IDL arg validation, luau-analyze/-Werror gates, CMake, SBOM, fuzzers (61,78,79,
+97,98,99,100).
+
+Environment note: the full `make test-sanitize` builds ~70 statically-linked
+ASan binaries (~19GB) which EXCEEDS the sandbox's free disk (~19GB on a 25GB
+volume). Run the sanitizer per-suite (`make build/san/test_test_X && ./build/san/test_test_X test`) and delete each binary, or it dies with "No space left on device".
+
+Not implemented: impr 59 (DAP) — the sandboxed Luau VM exposes no line-level
+debug hook by design. Impr 17 is memchr+Horspool, not full SSE2.

.openhands/memory/MEMORY.md

@@ -0,0 +1,21 @@
+# BerryAPI — project memory
+
+## Build / test
+- `make test` rebuilds vendored mbedTLS + Luau from scratch the first time: ~4-6 min on 4 cores. Run it in the background and poll a log file.
+- `make test-sanitize` (ASan+UBSan) takes ~9-10 min cold. Both must stay green.
+- Every `tests/test_*.cpp` is its own binary `build/test_test_*`; build/run one with `make build/test_test_<name> && ./build/test_test_<name> test`.
+- Adding a test file requires a `tools/sources.mk` TEST_SRC entry (`make check-sources` fails otherwise).
+- Header dep tracking: the Makefile `-include`s `*.d`; `build/tools/*.d` for berry-run/berry-relay/berry-sigcheck must be listed too, or a header change leaves a stale tool object that silently mismatches the core (fixed in this branch).
+- `check-run` (part of `make test`) drives `build/berry-run` end to end.
+
+## Conventions learned
+- Git author for this repo's work: `Lynxifyy <297587416+Lynxifyy@users.noreply.github.com>` (repo default was `openhands`).
+- PRs: `create_pr` / GitHub REST returns 403; `GH_TOKEN="$Github" gh pr create` works.
+- Commits on this branch use no Co-authored-by (user directive).
+
+## Landmines
+- The shim hotpatch backend is x86-64 only; a `jmp rel32` patch and its trampoline must be within +/-2 GiB, so the test must keep target, detour and any RIP-referenced global in one mmap'd page.
+- `Berry::u8/u16/u32/u64` are in namespace Berry; a .cpp in the global namespace needs explicit `using Berry::u16;` etc.
+- `Signature::Parse` now rejects a lone hex digit and supports half-nibble / IDA / \x styles; `wild` holds the per-byte fixed-bit mask.
+- `ResolveAdrpAdd(pc, rd, maxInsns)` — always pass the caller's remaining instruction count so the look-ahead cannot read past a buffer.
+- `core/net/net.cpp` includes `berry/voice_crypto.h` for `VoiceRandomBytes` (the CSPRNG behind WS keys/masks).

CMakeLists.txt

@@ -0,0 +1,97 @@
+# BerryAPI CMake build (improvement 99).
+#
+# An alternative to the Makefile, for a developer who wants CMake + Ninja +
+# ccache. It builds the same artefacts: the host test binaries, the injectable
+# shared library, and the tools. The source lists come from tools/sources.mk
+# (the single source of truth the Makefile and the Android build also read), so
+# the two builds cannot drift.
+#
+#   cmake -S . -B build-cmake -G Ninja -DCMAKE_BUILD_TYPE=Release
+#   cmake --build build-cmake
+#   ctest --test-dir build-cmake
+#
+# `make test` remains the canonical gate (it also runs the IDL, signer and
+# provenance checks); this target is for a fast incremental edit/build loop.
+
+cmake_minimum_required(VERSION 3.16)
+project(BerryAPI CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
+
+# --- parse tools/sources.mk for the canonical source lists ------------------
+# A tiny CMake script extracts CORE_SRC / TEST_SRC and the vendored lists, so
+# adding a file to sources.mk is picked up here too.
+function(load_sources_mk)
+  file(READ "${CMAKE_SOURCE_DIR}/tools/sources.mk" _mk)
+  # Collapse line continuations, then pull each VAR := ... block.
+  string(REGEX REPLACE "\\\\\n" " " _mk "${_mk}")
+  foreach(_var CORE_SRC TEST_SRC ED25519_SRC LZ4_SRC MBEDTLS_SRC)
+    string(REGEX MATCH "${_var} := ([^\n]*)" _m "${_mk}")
+    set(_list "${CMAKE_MATCH_1}")
+    separate_arguments(_list)
+    set(${_var} "${_list}" PARENT_SCOPE)
+  endforeach()
+endfunction()
+load_sources_mk()
+
+set(BERRY_INCLUDES
+    ${CMAKE_SOURCE_DIR}/include
+    ${CMAKE_SOURCE_DIR}/third_party/ed25519
+    ${CMAKE_SOURCE_DIR}/third_party/lz4/lib
+    ${CMAKE_SOURCE_DIR}/third_party/luau/Common/include
+    ${CMAKE_SOURCE_DIR}/third_party/luau/Ast/include
+    ${CMAKE_SOURCE_DIR}/third_party/luau/Compiler/include
+    ${CMAKE_SOURCE_DIR}/third_party/luau/VM/include
+    ${CMAKE_SOURCE_DIR}/third_party/mbedtls/include)
+add_compile_definitions(MBEDTLS_CONFIG_FILE="mbedtls/mbedtls_config.h")
+
+# Vendored Luau sources (the VM + compiler + AST).
+file(GLOB_RECURSE LUAU_SRC CONFIGURE_DEPENDS
+     ${CMAKE_SOURCE_DIR}/third_party/luau/Ast/src/*.cpp
+     ${CMAKE_SOURCE_DIR}/third_party/luau/Compiler/src/*.cpp
+     ${CMAKE_SOURCE_DIR}/third_party/luau/VM/src/*.cpp)
+
+# The core library (everything in CORE_SRC plus the vendored C).
+add_library(berry_core STATIC
+    ${CORE_SRC} ${ED25519_SRC} ${LZ4_SRC} ${MBEDTLS_SRC} ${LUAU_SRC})
+target_include_directories(berry_core PUBLIC ${BERRY_INCLUDES})
+target_compile_options(berry_core PRIVATE -Wall -Wextra)
+# Vendored code is not warning-clean under our flags; silence it there only.
+set_source_files_properties(${LUAU_SRC} ${LZ4_SRC} ${MBEDTLS_SRC}
+    PROPERTIES COMPILE_OPTIONS "-w")
+find_package(Threads REQUIRED)
+target_link_libraries(berry_core PUBLIC Threads::Threads dl)
+
+# The injectable shared library.
+add_library(BerryAPI SHARED ${CORE_SRC} ${ED25519_SRC} ${LZ4_SRC}
+            ${MBEDTLS_SRC} ${LUAU_SRC})
+target_include_directories(BerryAPI PUBLIC ${BERRY_INCLUDES})
+target_compile_options(BerryAPI PRIVATE -fPIC -fvisibility=hidden -DBERRY_BUILD_SHARED)
+set_target_properties(BerryAPI PROPERTIES
+    CXX_VISIBILITY_PRESET hidden
+    VERSION ${PROJECT_VERSION})
+
+# One test binary per tests/test_*.cpp, mirroring the Makefile.
+foreach(_test_src ${TEST_SRC})
+  get_filename_component(_name ${_test_src} NAME_WE)
+  add_executable(${_name} ${_test_src})
+  target_include_directories(${_name} PRIVATE ${CMAKE_SOURCE_DIR}/tests)
+  target_link_libraries(${_name} PRIVATE berry_core)
+  add_test(NAME ${_name} COMMAND ${_name} ${_name})
+endforeach()
+
+# The tools.
+foreach(_tool berry-run berry-relay berry-sigcheck)
+  if(EXISTS ${CMAKE_SOURCE_DIR}/tools/${_tool}.cpp)
+    add_executable(${_tool} tools/${_tool}.cpp)
+    target_include_directories(${_tool} PRIVATE ${CMAKE_SOURCE_DIR}/tools)
+    target_link_libraries(${_tool} PRIVATE berry_core)
+  endif()
+endforeach()