Refactor

CMakeLists.txt

@@ -36,10 +36,15 @@ if(GGML_CUDA)
     set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE}
         ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_cuda_ext_hook.py <SOURCE_DIR>)
 endif()
+# Overridable so a regression can be bisected against another tag in a separate
+# build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch
+# anchors in scripts/patch_ggml_cuda_ext_hook.py are checked against the default.
+set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch")
+
 include(FetchContent)
 FetchContent_Declare(llama
     GIT_REPOSITORY https://github.com/ggml-org/llama.cpp
-    GIT_TAG        b10331
+    GIT_TAG        ${VLA_LLAMA_TAG}
     GIT_SHALLOW    TRUE
     ${_vla_llama_patch}
 )
@@ -64,6 +69,14 @@ vla_exclude_fetched_targets(${llama_SOURCE_DIR})
 
 add_library(vla_core
     src/model.cpp
+    src/loader.cpp
+    src/options.cpp
+    src/modules/action_expert.cpp
+    src/modules/dit_head.cpp
+    src/modules/encoder.cpp
+    src/modules/prompt.cpp
+    src/modules/qwen3_lm.cpp
+    src/modules/siglip_vit.cpp
     src/models/smolvla.cpp
     src/models/pi0.cpp
     src/models/pi05.cpp
@@ -114,6 +127,9 @@ if(GGML_CUDA)
         CUDA_SEPARABLE_COMPILATION ON
         POSITION_INDEPENDENT_CODE ON
     )
+    target_include_directories(bitvla_cuda_kernels PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+    )
     target_compile_features(bitvla_cuda_kernels PRIVATE cxx_std_17)
     target_compile_options(bitvla_cuda_kernels PRIVATE
         $<$<COMPILE_LANGUAGE:CUDA>:-O3 --use_fast_math -Xptxas=-O3>

ci/lib/common.sh

@@ -139,7 +139,7 @@ apply_gr00t_env() {
     local arch="$1"
     case "$arch" in
         gr00t_n1_5|gr00t_n1_6|gr00t_n1_7)
-            export VLA_GR00T_BF16_WEIGHTS="${VLA_GR00T_BF16_WEIGHTS:-1}" ;;
+            : ;;
     esac
     case "$arch" in
         gr00t_n1_5) export VLA_GR00T_EMBODIMENT="${VLA_GR00T_EMBODIMENT:-new_embodiment}" ;;

docs/backend/sycl.md

@@ -217,6 +217,6 @@ weights) does not fit and dies in the allocator:
 level_zero backend failed with error: 38 (UR_RESULT_ERROR_OUT_OF_HOST_MEMORY)
 ```
 
-`VLA_GR00T_BF16_WEIGHTS=1` halves the weights but its activations still overflow
+`--weight-dtype bf16` (now the default) halves the weights but its activations still overflow
 the card. There is no host-memory spill path - the core is single-backend - so
 the larger checkpoints need an A770/B580-class card or better.

eval/README.md

@@ -56,7 +56,7 @@ Notes:
   accept the licence, or point `--tokenizer` at a local copy.
 - **GR00T** arches need `--stats-json <ckpt>/dataset_statistics.json` and an embodiment selected
   server-side via `VLA_GR00T_EMBODIMENT` (`new_embodiment` for N1.5, `libero_panda` for N1.6,
-  `libero_sim` for N1.7), plus `VLA_GR00T_BF16_WEIGHTS=1` to fit an 8 GB card.
+  `libero_sim` for N1.7). BF16 weights are the default, which is also what fits an 8 GB card.
 
 To sweep every model over `libero_object` tasks 0–9, use `eval/run_libero.sh -i <MODELS_ROOT>`.
 
@@ -66,7 +66,7 @@ So far only **GR00T-N1.6** is wired (the `gr00t-n1d6-bridge` checkpoint with the
 embodiment). Serve it, then drive from the SimplerEnv venv:
 
 ```bash
-VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=oxe_widowx \
+VLA_GR00T_EMBODIMENT=oxe_widowx \
     ./build/vla-server "$GR00T_N1D6_GGUF"
 
 eval/sim/simpler/simpler_uv/.venv/bin/python eval/client/run_simpler_client_direct.py \

eval/refactor_verify.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Copyright 2026 VinRobotics - Apache-2.0
+#
+# Bit-exactness and latency harness for the src/ layer/module/model refactor.
+#
+#   eval/refactor_verify.sh <outdir>            actions only
+#   BENCH=20 eval/refactor_verify.sh <outdir>   actions + predict() timing
+#
+# Each arch runs twice: at its shipping defaults, and under the alternate
+# precision. Both must stay byte-identical across a refactor, and neither may
+# regress in latency.
+#
+#   eval/refactor_verify.sh outputs/refactor/before
+#   ...change...
+#   cmake --build build -j"$(nproc)" --target vla_predict_check
+#   eval/refactor_verify.sh outputs/refactor/after
+#   diff -r outputs/refactor/before outputs/refactor/after
+#
+# Never rebuild while a sweep is running: relinking libvla_core.so under it
+# makes every remaining arch fail to load.
+#
+# ARCHS=... restricts the sweep. The square input side is probed rather than
+# hardcoded, because a tower fed the wrong side returns action_len=0 instead of
+# failing, and a wrong side would silently "pass" a diff.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+BIN="${BIN:-${REPO_ROOT}/build/tests/vla_predict_check}"
+HF="${HF:-/mnt/data/hf_data/vrfai}"
+OUT="${1:-${REPO_ROOT}/outputs/refactor/baseline}"
+SIDES="${SIDES:-224 256 448 512}"
+BENCH="${BENCH:-0}"
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
+
+# arch|ckpt|mmproj|n_images|env|alternate-config CLI flags.
+# openvla_oft has no alternate: at f32 its weights need 30 GB.
+MODELS=(
+    "smolvla|${HF}/smolvla-libero-gguf/smolvla-libero.gguf|${HF}/backup/mmproj-smolvla-libero.gguf|2||--flash-attn --mm-prec default"
+    "pi0|${HF}/pi0-libero-finetuned-v044-gguf/pi0-libero-finetuned-v044.gguf|${HF}/backup/mmproj-pi0-libero-finetuned-v044.gguf|2||--act-dtype bf16 --flash-attn"
+    "pi05|${HF}/pi05-libero-gguf/pi05-libero.gguf|${HF}/backup/mmproj-pi05-libero.gguf|2||--weight-dtype f32"
+    "evo1|${HF}/evo1-libero-gguf/evo1-libero.gguf||2||--act-dtype bf16 --flash-attn"
+    "gr00t_n1_5|${HF}/gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf||2||--weight-dtype f32"
+    "gr00t_n1_6|${HF}/gr00tn1d6-libero-gguf/gr00tn1d6-libero.gguf||2||--weight-dtype f32"
+    "gr00t_n1_7|${HF}/gr00tn1d7-libero-gguf/libero_object/gr00tn1d7-libero-object.gguf||2||--weight-dtype f32"
+    "bitvla|${HF}/bitvla-libero-gguf/libero_object/bitvla-libero-object.gguf||2||--weight-dtype bf16"
+    "vla_adapter|${HF}/vla-adapter-libero-object-gguf/libero_object/vla-adapter-libero-object.gguf||2||--weight-dtype f32"
+    "openvla_oft|${HF}/openvla-oft-libero-gguf/openvla-oft-libero.gguf||2||"
+    "vla_jepa|${HF}/vla-jepa-libero/vla-jepa.gguf||2|VLA_EXTRA_TOKEN=151697 VLA_EXTRA_COUNT=32|--weight-dtype f32"
+)
+
+[[ -x "${BIN}" ]] || { echo "ERROR: missing ${BIN} (cmake -DVLA_BUILD_TESTS=ON)" >&2; exit 1; }
+mkdir -p "${OUT}"
+
+run_one() {
+    local arch="$1" ckpt="$2" mmproj="$3" nimg="$4" env_str="$5" tag="$6" side="$7" cli="$8"
+    # shellcheck disable=SC2086
+    env ${env_str} VLA_IMG_SIZE="${side}" VLA_BENCH_ITERS="${BENCH}" \
+        "${BIN}" "${ckpt}" "${mmproj}" "${nimg}" ${cli} \
+        > "${OUT}/${arch}${tag}.actions.txt" 2> "${OUT}/${arch}${tag}.log"
+}
+
+fail=0
+for row in "${MODELS[@]}"; do
+    IFS='|' read -r arch ckpt mmproj nimg always fastest <<< "${row}"
+
+    if [[ -n "${ARCHS:-}" && " ${ARCHS} " != *" ${arch} "* ]]; then
+        continue
+    fi
+    if [[ ! -e "${ckpt}" ]]; then
+        echo "[skip] ${arch}: no checkpoint at ${ckpt}"
+        continue
+    fi
+
+    side=""
+    for s in ${SIDES}; do
+        if run_one "${arch}" "${ckpt}" "${mmproj}" "${nimg}" "${always}" "" "${s}" "" \
+           && ! grep -q '^action_len=0$' "${OUT}/${arch}.actions.txt"; then
+            side="${s}"
+            echo "${s}" > "${OUT}/${arch}.side"
+            break
+        fi
+    done
+    if [[ -z "${side}" ]]; then
+        echo "[FAIL] ${arch}: no input side in '${SIDES}' produced a chunk; see ${OUT}/${arch}.log" >&2
+        fail=1
+        continue
+    fi
+
+    line="[ok  ] ${arch}  side=${side}"
+    [[ "${BENCH}" -gt 0 ]] && line+="  default=$(grep -oP 'min=\K[0-9.]+' "${OUT}/${arch}.log" | head -1)ms"
+
+    if [[ -n "${fastest}" ]]; then
+        if run_one "${arch}" "${ckpt}" "${mmproj}" "${nimg}" "${always}" ".alt" "${side}" "${fastest}" \
+           && ! grep -q '^action_len=0$' "${OUT}/${arch}.alt.actions.txt"; then
+            line+="  alt=ok"
+            [[ "${BENCH}" -gt 0 ]] && line+=" $(grep -oP 'min=\K[0-9.]+' "${OUT}/${arch}.alt.log" | head -1)ms"
+        else
+            echo "[FAIL] ${arch}: alternate config produced no chunk; see ${OUT}/${arch}.fast.log" >&2
+            fail=1
+        fi
+    fi
+    echo "${line}"
+done
+
+echo
+echo "written to ${OUT}"
+exit "${fail}"

eval/run_libero.sh

@@ -298,9 +298,10 @@ run_model() {
         client_extra+=(--stats-json "${stats_json}")
     fi
 
+    # BF16 weights are the shipping default now, and are what every published
+    # GR00T success rate was measured under; passed explicitly so the log records it.
     if [[ "${arch}" == gr00t_n1_5 || "${arch}" == gr00t_n1_6 || "${arch}" == gr00t_n1_7 ]]; then
-        export VLA_GR00T_BF16_WEIGHTS="${VLA_GR00T_BF16_WEIGHTS:-1}"
-        echo "[${arch}] VLA_GR00T_BF16_WEIGHTS=${VLA_GR00T_BF16_WEIGHTS}"
+        server_args+=(--weight-dtype "${WEIGHT_DTYPE:-bf16}")
     fi
     if [[ -n "${_USER_VLA_GR00T_EMBODIMENT}" ]]; then
         export VLA_GR00T_EMBODIMENT="${_USER_VLA_GR00T_EMBODIMENT}"

eval/run_libero_client.sh

@@ -142,21 +142,20 @@ EOF
             ;;
         gr00t_n1_5)
             cat <<EOF
-    VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=new_embodiment \\
+    VLA_GR00T_EMBODIMENT=new_embodiment \\
     ./build/vla-server --bind tcp://*:${PORT} \\
         \${MODELS_ROOT}/gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf
 EOF
             ;;
         gr00t_n1_6)
             cat <<EOF
-    VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=libero_panda \\
+    VLA_GR00T_EMBODIMENT=libero_panda \\
     ./build/vla-server --bind tcp://*:${PORT} \\
         \${MODELS_ROOT}/gr00t-n1d6-libero-gguf/gr00t-n1d6-libero.gguf
 EOF
             ;;
         gr00t_n1_7)
             cat <<EOF
-    VLA_GR00T_BF16_WEIGHTS=1 \\
     ./build/vla-server --bind tcp://*:${PORT} \\
         \${MODELS_ROOT}/gr00t-n1d7-libero-object-gguf/gr00t-n1d7-libero-object.gguf
 EOF

eval/run_libero_server.sh

@@ -45,7 +45,7 @@ Usage: $(basename "$0") -m MODEL [-i MODELS_ROOT] [-a BIND_ADDR] [-o LOG_DIR] [-
   -B               skip the cmake build (use the existing vla-server binary)
   -h               show this help
 
-Env overrides: BIND_ADDR, SERVER_BIN, VLA_GR00T_BF16_WEIGHTS, VLA_GR00T_EMBODIMENT
+Env overrides: BIND_ADDR, SERVER_BIN, VLA_GR00T_EMBODIMENT
 EOF
 }
 
@@ -143,13 +143,9 @@ case "${MODEL}" in
 esac
 
 # GR00T env. Honour a user-supplied VLA_GR00T_EMBODIMENT; otherwise default per
-# arch (matches eval/run_libero.sh). VLA_GR00T_BF16_WEIGHTS defaults to 1 (BF16 is
-# the smaller weight path - important on the Nano's 8 GB unified RAM).
+# arch (matches eval/run_libero.sh). BF16 weights are the shipping default and
+# the smaller weight path - important on the Nano's 8 GB unified RAM.
 _USER_VLA_GR00T_EMBODIMENT="${VLA_GR00T_EMBODIMENT-}"
-if [[ "${MODEL}" == gr00t_n1_5 || "${MODEL}" == gr00t_n1_6 || "${MODEL}" == gr00t_n1_7 ]]; then
-    export VLA_GR00T_BF16_WEIGHTS="${VLA_GR00T_BF16_WEIGHTS:-1}"
-    echo "[${MODEL}] VLA_GR00T_BF16_WEIGHTS=${VLA_GR00T_BF16_WEIGHTS}"
-fi
 if [[ -n "${_USER_VLA_GR00T_EMBODIMENT}" ]]; then
     export VLA_GR00T_EMBODIMENT="${_USER_VLA_GR00T_EMBODIMENT}"
 else

eval/run_simpler.sh

@@ -293,10 +293,8 @@ run_model() {
         echo "[skip] ${arch}: statistics.json not found at ${stats_json} (set GR00T_N1_6_BRIDGE_STATS)" >&2
         return 1
     fi
-
-    export VLA_GR00T_BF16_WEIGHTS="${VLA_GR00T_BF16_WEIGHTS:-1}"
+    # bf16 weights are the shipping default; the env switch was retired.
     export VLA_GR00T_EMBODIMENT="${VLA_GR00T_EMBODIMENT:-${EMBODIMENT}}"
-    echo "[${arch}] VLA_GR00T_BF16_WEIGHTS=${VLA_GR00T_BF16_WEIGHTS}  VLA_GR00T_EMBODIMENT=${VLA_GR00T_EMBODIMENT}"
 
     local log="${LOG_DIR}/${arch}.log"
     echo "===================="

scripts/patch_ggml_cuda_ext_hook.py

@@ -30,13 +30,19 @@
 
 What it changes (ggml/src/ggml-cuda/ggml-cuda.cu only)
 ------------------------------------------------------
-  1. An exported function pointer, null by default.
-  2. One call to it at the top of ggml_cuda_compute_forward. Returning false
-     means "not mine", and ggml runs the op exactly as before.
+  1. Two exported function pointers, null by default.
+  2. One call to the first at the top of ggml_cuda_compute_forward. Returning
+     false means "not mine", and ggml runs the op exactly as before.
   3. The RMS_NORM+MUL fusion check GGML_ASSERTs F32 rather than declining, so a
      BF16 rms_norm aborts the process before dispatch is ever reached. Those two
      asserts become a return, which is what the surrounding checks already do
      for every other unsupported type.
+  4. One call to the second in the ADD/MUL fusion branch of ggml_cuda_try_fuse.
+     Fusion happens in ggml_backend_cuda_graph_compute, upstream of
+     ggml_cuda_compute_forward, so the hook in (2) never sees a fused node --
+     and ggml_cuda_op_fused_binbcast_impl handles F32/F16 only and GGML_ABORTs
+     on BF16. Without this the choice is a crash or no fusion at all for BF16
+     activations, and the unfused path costs ~18 ms/call on evo1.
 
 With the pointer left null this is a no-op, so an unpatched-but-hooked ggml
 behaves identically to a stock one.
@@ -57,6 +63,11 @@
 extern "C" {
 typedef bool (*ggml_cuda_ext_forward_t)(struct ggml_tensor * dst, void * stream);
 __attribute__((visibility("default"))) ggml_cuda_ext_forward_t ggml_cuda_ext_forward = nullptr;
+
+// Same contract for a fused ADD/MUL run: dst carries src[0] plus n_fuse addends
+// in src[1..n_fuse], all sharing one layout, and dst->data is the final output.
+typedef bool (*ggml_cuda_ext_fused_binbcast_t)(struct ggml_tensor * dst, int n_fuse, void * stream);
+__attribute__((visibility("default"))) ggml_cuda_ext_fused_binbcast_t ggml_cuda_ext_fused_binbcast = nullptr;
 }
 
 static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) {
@@ -80,6 +91,25 @@
         }""",
 )
 
+# The fused ADD/MUL run is assembled here and handed to a kernel that supports
+# F32/F16 only. Offer it to the extension first; declining costs one null check.
+FUSED_BINBCAST_GUARD = (
+    """            if (node->op == GGML_OP_ADD) {
+                ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse);
+            } else {
+                ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse);
+            }""",
+    """            // vla.cpp: CUDA extension hook - first refusal on the fused node.
+            if (!(ggml_cuda_ext_fused_binbcast &&
+                  ggml_cuda_ext_fused_binbcast(&fused_node, n_fuse, (void *) cuda_ctx->stream()))) {
+                if (node->op == GGML_OP_ADD) {
+                    ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse);
+                } else {
+                    ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse);
+                }
+            }""",
+)
+
 
 def main():
     src = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve()
@@ -91,7 +121,7 @@ def main():
     if MARKER in text:
         return  # idempotent: re-configure over an already-patched tree
 
-    for old, new in (HOOK_DECL, FUSION_GUARD):
+    for old, new in (HOOK_DECL, FUSION_GUARD, FUSED_BINBCAST_GUARD):
         n = text.count(old)
         if n != 1:
             raise SystemExit(