From ad9a56b58a62e3e5ee2d6fae6e800be2c83334fb Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 12:34:39 +0700 Subject: [PATCH 01/21] give the cuda extension first refusal on fused add/mul so bf16 activations stop aborting --- scripts/patch_ggml_cuda_ext_hook.py | 38 ++++++++- src/cuda/vla_cuda_bf16.cu | 124 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 4 deletions(-) diff --git a/scripts/patch_ggml_cuda_ext_hook.py b/scripts/patch_ggml_cuda_ext_hook.py index dff4bd5..d02bcb4 100644 --- a/scripts/patch_ggml_cuda_ext_hook.py +++ b/scripts/patch_ggml_cuda_ext_hook.py @@ -30,13 +30,19 @@ What it changes (ggml/src/ggml-cuda/ggml-cuda.cu only) ------------------------------------------------------ - 1. An exported function pointer, null by default. - 2. One call to it at the top of ggml_cuda_compute_forward. Returning false - means "not mine", and ggml runs the op exactly as before. + 1. Two exported function pointers, null by default. + 2. One call to the first at the top of ggml_cuda_compute_forward. Returning + false means "not mine", and ggml runs the op exactly as before. 3. The RMS_NORM+MUL fusion check GGML_ASSERTs F32 rather than declining, so a BF16 rms_norm aborts the process before dispatch is ever reached. Those two asserts become a return, which is what the surrounding checks already do for every other unsupported type. + 4. One call to the second in the ADD/MUL fusion branch of ggml_cuda_try_fuse. + Fusion happens in ggml_backend_cuda_graph_compute, upstream of + ggml_cuda_compute_forward, so the hook in (2) never sees a fused node -- + and ggml_cuda_op_fused_binbcast_impl handles F32/F16 only and GGML_ABORTs + on BF16. Without this the choice is a crash or no fusion at all for BF16 + activations, and the unfused path costs ~18 ms/call on evo1. With the pointer left null this is a no-op, so an unpatched-but-hooked ggml behaves identically to a stock one. @@ -57,6 +63,11 @@ extern "C" { typedef bool (*ggml_cuda_ext_forward_t)(struct ggml_tensor * dst, void * stream); __attribute__((visibility("default"))) ggml_cuda_ext_forward_t ggml_cuda_ext_forward = nullptr; + +// Same contract for a fused ADD/MUL run: dst carries src[0] plus n_fuse addends +// in src[1..n_fuse], all sharing one layout, and dst->data is the final output. +typedef bool (*ggml_cuda_ext_fused_binbcast_t)(struct ggml_tensor * dst, int n_fuse, void * stream); +__attribute__((visibility("default"))) ggml_cuda_ext_fused_binbcast_t ggml_cuda_ext_fused_binbcast = nullptr; } static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) { @@ -80,6 +91,25 @@ }""", ) +# The fused ADD/MUL run is assembled here and handed to a kernel that supports +# F32/F16 only. Offer it to the extension first; declining costs one null check. +FUSED_BINBCAST_GUARD = ( + """ if (node->op == GGML_OP_ADD) { + ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse); + } else { + ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse); + }""", + """ // vla.cpp: CUDA extension hook - first refusal on the fused node. + if (!(ggml_cuda_ext_fused_binbcast && + ggml_cuda_ext_fused_binbcast(&fused_node, n_fuse, (void *) cuda_ctx->stream()))) { + if (node->op == GGML_OP_ADD) { + ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse); + } else { + ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse); + } + }""", +) + def main(): src = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve() @@ -91,7 +121,7 @@ def main(): if MARKER in text: return # idempotent: re-configure over an already-patched tree - for old, new in (HOOK_DECL, FUSION_GUARD): + for old, new in (HOOK_DECL, FUSION_GUARD, FUSED_BINBCAST_GUARD): n = text.count(old) if n != 1: raise SystemExit( diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index 6ff6b2f..ccc61f1 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -48,6 +48,9 @@ extern "C" { typedef bool (*ggml_cuda_ext_forward_t)(struct ggml_tensor * dst, void * stream); extern ggml_cuda_ext_forward_t ggml_cuda_ext_forward; + +typedef bool (*ggml_cuda_ext_fused_binbcast_t)(struct ggml_tensor * dst, int n_fuse, void * stream); +extern ggml_cuda_ext_fused_binbcast_t ggml_cuda_ext_fused_binbcast; } namespace { @@ -131,6 +134,108 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { return true; } +// --------------------------------------------------------------------------- +// fused elementwise binary: dst = (((src0 op src1) op src2) ... op src[n_fuse]) +// --------------------------------------------------------------------------- +// +// ggml fuses runs of up to 8 ADD or MUL nodes in ggml_cuda_try_fuse and hands +// the run over as one synthetic node: src[0] is the base, src[1..n_fuse] are the +// addends, dst->data is the last node's output. The fusion check upstream only +// admits a run when every addend has the same layout, which is why one stride +// set covers all of them. +// +// Accumulation is in float with a single conversion at the end, matching ggml's +// own k_bin_bcast: `float result = (float) src0[..]; result = bin_op(result, ..); +// dst[i0] = (dst_t) result;`. Doing it any other way would make a fused run +// disagree with the unfused one. + +constexpr int MAX_FUSE = 8; + +template +struct SrcPtrs { const S1 * p[MAX_FUSE]; }; + +template +__global__ void k_fused_bin_bcast_bf16( + const __nv_bfloat16 * __restrict__ src0, const SrcPtrs srcs, const int n_fuse, + __nv_bfloat16 * __restrict__ dst, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t s10, const int64_t s11, const int64_t s12, const int64_t s13, + const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { + const int64_t total = ne0*ne1*ne2*ne3; + for (int64_t idx = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; idx < total; + idx += (int64_t) gridDim.x*blockDim.x) { + const int64_t i0 = idx % ne0; + const int64_t i1 = (idx / ne0) % ne1; + const int64_t i2 = (idx / (ne0*ne1)) % ne2; + const int64_t i3 = idx / (ne0*ne1*ne2); + + const int64_t j = (i0 % ne10)*s10 + (i1 % ne11)*s11 + + (i2 % ne12)*s12 + (i3 % ne13)*s13; + + float acc = bf2f(src0[i0*s00 + i1*s01 + i2*s02 + i3*s03]); + for (int k = 0; k < n_fuse; ++k) { + const float b = (float) srcs.p[k][j]; + acc = (op == BinOp::Add) ? acc + b : acc * b; + } + dst[i0*d0 + i1*d1 + i2*d2 + i3*d3] = f2bf(acc); + } +} + +template +bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { + if (n_fuse < 2 || n_fuse > MAX_FUSE) return false; + + const ggml_tensor * src0 = dst->src[0]; + if (!src0) return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; + if (!ggml_are_same_shape(src0, dst)) return false; + + // src[1] fixes the layout and type every other addend must match; the + // upstream fusion check guarantees it, and this re-checks rather than + // trusting it, because getting it wrong reads out of bounds. + const ggml_tensor * src1 = dst->src[1]; + if (!src1) return false; + if (src1->type != GGML_TYPE_BF16 && src1->type != GGML_TYPE_F32) return false; + if (!ggml_can_repeat(src1, src0)) return false; + + for (int k = 1; k < n_fuse; ++k) { + const ggml_tensor * s = dst->src[k + 1]; + if (!s || s->type != src1->type) return false; + if (!ggml_are_same_shape(s, src1)) return false; + for (int d = 0; d < GGML_MAX_DIMS; ++d) { + if (s->nb[d] != src1->nb[d]) return false; + } + } + + const int64_t total = ggml_nelements(dst); + const int64_t blocks = (total + BLOCK - 1) / BLOCK; + const int grid = (int) (blocks < 65535 ? blocks : 65535); + +#define VLA_LAUNCH_FUSED(TYPE) \ + do { \ + SrcPtrs srcs{}; \ + for (int k = 0; k < n_fuse; ++k) srcs.p[k] = (const TYPE *) dst->src[k + 1]->data; \ + k_fused_bin_bcast_bf16<<>>( \ + (const __nv_bfloat16 *) src0->data, srcs, n_fuse, \ + (__nv_bfloat16 *) dst->data, \ + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], \ + es(src0,0), es(src0,1), es(src0,2), es(src0,3), \ + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], \ + es(src1,0), es(src1,1), es(src1,2), es(src1,3), \ + es(dst,0), es(dst,1), es(dst,2), es(dst,3)); \ + } while (0) + + if (src1->type == GGML_TYPE_BF16) { + VLA_LAUNCH_FUSED(__nv_bfloat16); + } else { + VLA_LAUNCH_FUSED(float); + } +#undef VLA_LAUNCH_FUSED + return true; +} + // --------------------------------------------------------------------------- // unary // --------------------------------------------------------------------------- @@ -340,6 +445,17 @@ bool mul_mat(ggml_tensor * dst, cudaStream_t stream) { // hook entry point // --------------------------------------------------------------------------- +extern "C" bool vla_cuda_bf16_fused_binbcast(ggml_tensor * dst, int n_fuse, void * stream_v) { + if (!dst) return false; + cudaStream_t stream = (cudaStream_t) stream_v; + + switch (dst->op) { + case GGML_OP_ADD: return fused_bin_bcast(dst, n_fuse, stream); + case GGML_OP_MUL: return fused_bin_bcast(dst, n_fuse, stream); + default: return false; + } +} + extern "C" bool vla_cuda_bf16_forward(ggml_tensor * dst, void * stream_v) { if (!dst) return false; cudaStream_t stream = (cudaStream_t) stream_v; @@ -368,6 +484,14 @@ namespace vla { // Called once, after the CUDA backend is up. Idempotent. void cuda_register_bf16_ops() { ggml_cuda_ext_forward = vla_cuda_bf16_forward; + + // Fusion runs in ggml_backend_cuda_graph_compute, upstream of + // ggml_cuda_compute_forward, so the pointer above never sees a fused node. + // Without this second one a fused BF16 add reaches a kernel that handles + // F32/F16 only and GGML_ABORTs ("unsupported types for fusion: dst: bf16, + // src0: bf16, src1: f32"), and declining fusion instead costs ~18 ms/call + // on evo1 -- enough to make BF16 activations a net loss. + ggml_cuda_ext_fused_binbcast = vla_cuda_bf16_fused_binbcast; } } // namespace vla From 0daf584b5628440af6de02e9b08cbb4498f68dbe Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 13:00:52 +0700 Subject: [PATCH 02/21] address bf16 elementwise rows by block index instead of a per-element divide --- src/cuda/vla_cuda_bf16.cu | 180 ++++++++++++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 38 deletions(-) diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index ccc61f1..c441fb4 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -70,8 +70,58 @@ inline __device__ __nv_bfloat16 f2bf(const float v) { return __float2bfloat16(v) enum class BinOp { Add, Mul }; +inline __device__ float apply_bin(BinOp op, float a, float b) { + return op == BinOp::Add ? a + b : a * b; +} + +// Broadcast index along one dimension. ggml's rule is a modulo, but the only +// shapes that occur are "same extent" and "extent 1"; both operands are kernel +// arguments so the branch is uniform across the block and the 64-bit modulo is +// left for the general case that never fires in practice. +inline __device__ int64_t bcast_idx(int64_t i, int64_t ne_src, int64_t ne_dst) { + if (ne_src == ne_dst) return i; + if (ne_src == 1) return 0; + return i % ne_src; +} + +// Rows are addressed through blockIdx.y/z rather than recovered from a flat +// index. A flat grid-stride loop costs a 64-bit division and three modulos per +// element to rebuild (i0,i1,i2,i3), which is what made this kernel 12.8 us/launch +// against ggml's 5.7 for the same work -- and at ~3,700 elementwise nodes per +// evo1 call that difference was larger than everything BF16 activations saved. +template +__global__ void k_bin_bcast_bf16_rows( + const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, + __nv_bfloat16 * __restrict__ dst, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t s10, const int64_t s11, const int64_t s12, const int64_t s13, + const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { + const int64_t i1 = blockIdx.y; + const int64_t i23 = blockIdx.z; + const int64_t i2 = i23 % ne2; // once per block, not per element + const int64_t i3 = i23 / ne2; + + const int64_t j1 = bcast_idx(i1, ne11, ne1); + const int64_t j2 = bcast_idx(i2, ne12, ne2); + const int64_t j3 = bcast_idx(i3, ne13, ne3); + + const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; + const S1 * __restrict__ r1 = src1 + j1*s11 + j2*s12 + j3*s13; + __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; + + for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i0 < ne0; + i0 += (int64_t) gridDim.x*blockDim.x) { + const float a = bf2f(r0[i0*s00]); + const float b = (float) r1[bcast_idx(i0, ne10, ne0)*s10]; + rd[i0*d0] = f2bf(apply_bin(op, a, b)); + } +} + +// Fallback for shapes the row grid cannot address (gridDim.y/z cap at 65535). template -__global__ void k_bin_bcast_bf16( +__global__ void k_bin_bcast_bf16_flat( const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, __nv_bfloat16 * __restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, @@ -91,13 +141,48 @@ __global__ void k_bin_bcast_bf16( const float b = (float) src1[(i0 % ne10)*s10 + (i1 % ne11)*s11 + (i2 % ne12)*s12 + (i3 % ne13)*s13]; - dst[i0*d0 + i1*d1 + i2*d2 + i3*d3] = f2bf(op == BinOp::Add ? a + b : a * b); + dst[i0*d0 + i1*d1 + i2*d2 + i3*d3] = f2bf(apply_bin(op, a, b)); } } // element strides (ggml stores byte strides) inline int64_t es(const ggml_tensor * t, int i) { return t->nb[i] / ggml_type_size(t->type); } +// Launch shape for the row-addressed kernels; ok=false means fall back to flat. +struct RowGrid { + dim3 grid; + dim3 block; + bool ok; +}; + +// VLA_BF16_FLAT=1 forces the flat kernel, for bisecting. The two paths do the +// same per-element arithmetic in the same order, so they are bit-identical -- +// which is what makes the A/B a correctness check rather than a smoke test. +inline bool force_flat() { + static const bool v = [] { + const char * s = std::getenv("VLA_BF16_FLAT"); + return s && *s && !(s[0] == '0' && s[1] == '\0'); + }(); + return v; +} + +inline RowGrid row_grid(int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { + RowGrid g{}; + const int64_t nz = ne2*ne3; + if (ne1 > 65535 || nz > 65535 || ne1 < 1 || nz < 1) { g.ok = false; return g; } + + unsigned bx = 32; + while (bx < (unsigned) BLOCK && (int64_t) bx < ne0) bx *= 2; + int64_t gx = (ne0 + bx - 1) / bx; + if (gx > 65535) gx = 65535; + if (gx < 1) gx = 1; + + g.block = dim3(bx, 1, 1); + g.grid = dim3((unsigned) gx, (unsigned) ne1, (unsigned) nz); + g.ok = true; + return g; +} + template bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; @@ -108,29 +193,42 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { if (!ggml_are_same_shape(src0, dst)) return false; if (!ggml_can_repeat(src1, src0)) return false; - const int64_t total = ggml_nelements(dst); - const int64_t blocks = (total + BLOCK - 1) / BLOCK; - const int grid = (int) (blocks < 65535 ? blocks : 65535); + // Only the unfused path honours VLA_BF16_FLAT: the fused path has no flat + // fallback, and declining there hands the run to ggml's aborting kernel. + const RowGrid g = force_flat() ? RowGrid{} + : row_grid(dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + +#define VLA_LAUNCH_BIN(TYPE) \ + do { \ + if (g.ok) { \ + k_bin_bcast_bf16_rows<<>>( \ + (const __nv_bfloat16 *) src0->data, (const TYPE *) src1->data, \ + (__nv_bfloat16 *) dst->data, \ + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], \ + es(src0,0), es(src0,1), es(src0,2), es(src0,3), \ + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], \ + es(src1,0), es(src1,1), es(src1,2), es(src1,3), \ + es(dst,0), es(dst,1), es(dst,2), es(dst,3)); \ + } else { \ + const int64_t blocks = (ggml_nelements(dst) + BLOCK - 1) / BLOCK; \ + const int flat = (int) (blocks < 65535 ? blocks : 65535); \ + k_bin_bcast_bf16_flat<<>>( \ + (const __nv_bfloat16 *) src0->data, (const TYPE *) src1->data, \ + (__nv_bfloat16 *) dst->data, \ + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], \ + es(src0,0), es(src0,1), es(src0,2), es(src0,3), \ + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], \ + es(src1,0), es(src1,1), es(src1,2), es(src1,3), \ + es(dst,0), es(dst,1), es(dst,2), es(dst,3)); \ + } \ + } while (0) if (src1->type == GGML_TYPE_BF16) { - k_bin_bcast_bf16<<>>( - (const __nv_bfloat16 *) src0->data, (const __nv_bfloat16 *) src1->data, - (__nv_bfloat16 *) dst->data, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - es(src0,0), es(src0,1), es(src0,2), es(src0,3), - src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], - es(src1,0), es(src1,1), es(src1,2), es(src1,3), - es(dst,0), es(dst,1), es(dst,2), es(dst,3)); + VLA_LAUNCH_BIN(__nv_bfloat16); } else { - k_bin_bcast_bf16<<>>( - (const __nv_bfloat16 *) src0->data, (const float *) src1->data, - (__nv_bfloat16 *) dst->data, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - es(src0,0), es(src0,1), es(src0,2), es(src0,3), - src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], - es(src1,0), es(src1,1), es(src1,2), es(src1,3), - es(dst,0), es(dst,1), es(dst,2), es(dst,3)); + VLA_LAUNCH_BIN(float); } +#undef VLA_LAUNCH_BIN return true; } @@ -163,23 +261,27 @@ __global__ void k_fused_bin_bcast_bf16( const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, const int64_t s10, const int64_t s11, const int64_t s12, const int64_t s13, const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { - const int64_t total = ne0*ne1*ne2*ne3; - for (int64_t idx = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; idx < total; - idx += (int64_t) gridDim.x*blockDim.x) { - const int64_t i0 = idx % ne0; - const int64_t i1 = (idx / ne0) % ne1; - const int64_t i2 = (idx / (ne0*ne1)) % ne2; - const int64_t i3 = idx / (ne0*ne1*ne2); + const int64_t i1 = blockIdx.y; + const int64_t i23 = blockIdx.z; + const int64_t i2 = i23 % ne2; + const int64_t i3 = i23 / ne2; + + const int64_t row1 = bcast_idx(i1, ne11, ne1)*s11 + + bcast_idx(i2, ne12, ne2)*s12 + + bcast_idx(i3, ne13, ne3)*s13; + + const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; + __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; - const int64_t j = (i0 % ne10)*s10 + (i1 % ne11)*s11 + - (i2 % ne12)*s12 + (i3 % ne13)*s13; + for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i0 < ne0; + i0 += (int64_t) gridDim.x*blockDim.x) { + const int64_t j = row1 + bcast_idx(i0, ne10, ne0)*s10; - float acc = bf2f(src0[i0*s00 + i1*s01 + i2*s02 + i3*s03]); + float acc = bf2f(r0[i0*s00]); for (int k = 0; k < n_fuse; ++k) { - const float b = (float) srcs.p[k][j]; - acc = (op == BinOp::Add) ? acc + b : acc * b; + acc = apply_bin(op, acc, (float) srcs.p[k][j]); } - dst[i0*d0 + i1*d1 + i2*d2 + i3*d3] = f2bf(acc); + rd[i0*d0] = f2bf(acc); } } @@ -209,15 +311,17 @@ bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { } } - const int64_t total = ggml_nelements(dst); - const int64_t blocks = (total + BLOCK - 1) / BLOCK; - const int grid = (int) (blocks < 65535 ? blocks : 65535); + // No flat fallback here: declining just sends the run to ggml's fused + // kernel, which aborts on BF16, so an unaddressable shape must decline + // before the hook claims it. + const RowGrid g = row_grid(dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + if (!g.ok) return false; #define VLA_LAUNCH_FUSED(TYPE) \ do { \ SrcPtrs srcs{}; \ for (int k = 0; k < n_fuse; ++k) srcs.p[k] = (const TYPE *) dst->src[k + 1]->data; \ - k_fused_bin_bcast_bf16<<>>( \ + k_fused_bin_bcast_bf16<<>>( \ (const __nv_bfloat16 *) src0->data, srcs, n_fuse, \ (__nv_bfloat16 *) dst->data, \ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], \ From 3e802decfaed3d32135365107874d579d764eb25 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 14:23:23 +0700 Subject: [PATCH 03/21] make the llama.cpp tag and server binary overridable so regressions can be bisected across builds --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54fcca6..fcbe4a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,10 +36,15 @@ if(GGML_CUDA) set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_cuda_ext_hook.py ) endif() +# Overridable so a regression can be bisected against another tag in a separate +# build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch +# anchors in scripts/patch_ggml_cuda_ext_hook.py are checked against the default. +set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch") + include(FetchContent) FetchContent_Declare(llama GIT_REPOSITORY https://github.com/ggml-org/llama.cpp - GIT_TAG b10331 + GIT_TAG ${VLA_LLAMA_TAG} GIT_SHALLOW TRUE ${_vla_llama_patch} ) From 1efb7a3a9a322c36c6aa874365f01d4f4cebc8e1 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 14:29:03 +0700 Subject: [PATCH 04/21] read boolean env switches by value so setting one to 0 turns it off --- CMakeLists.txt | 3 ++ src/cuda/vla_cuda_bf16.cu | 6 +-- src/env_flag.h | 61 ++++++++++++++++++++++++++++ src/kernels/bitvla/bitnet_kernels.cu | 4 +- src/models/bitvla.cpp | 9 ++-- src/models/evo1.cpp | 7 ++-- src/models/gr00tn1d5.cpp | 3 +- src/models/gr00tn1d6.cpp | 3 +- src/models/gr00tn1d7.cpp | 3 +- src/models/openvla_oft.cpp | 3 +- src/models/pi0.cpp | 7 ++-- src/models/pi05.cpp | 5 ++- src/models/qwen3vl_vit.h | 3 +- src/models/smolvla.cpp | 3 +- src/models/vla_adapter.cpp | 3 +- src/models/vla_jepa.cpp | 3 +- 16 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 src/env_flag.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fcbe4a5..8119a6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,9 @@ if(GGML_CUDA) CUDA_SEPARABLE_COMPILATION ON POSITION_INDEPENDENT_CODE ON ) + target_include_directories(bitvla_cuda_kernels PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ) target_compile_features(bitvla_cuda_kernels PRIVATE cxx_std_17) target_compile_options(bitvla_cuda_kernels PRIVATE $<$:-O3 --use_fast_math -Xptxas=-O3> diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index c441fb4..8390a83 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -36,6 +36,7 @@ // never a reduction. #include "ggml.h" +#include "env_flag.h" #include #include @@ -159,10 +160,7 @@ struct RowGrid { // same per-element arithmetic in the same order, so they are bit-identical -- // which is what makes the A/B a correctness check rather than a smoke test. inline bool force_flat() { - static const bool v = [] { - const char * s = std::getenv("VLA_BF16_FLAT"); - return s && *s && !(s[0] == '0' && s[1] == '\0'); - }(); + static const bool v = vla::env_flag("VLA_BF16_FLAT"); return v; } diff --git a/src/env_flag.h b/src/env_flag.h new file mode 100644 index 0000000..f7aa6ba --- /dev/null +++ b/src/env_flag.h @@ -0,0 +1,61 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file env_flag.h + * @brief Boolean environment switches with a value, not just presence. + */ + +#pragma once + +#include +#include +#include + +namespace vla { + +/** + * @brief Read a boolean environment switch. + * + * Presence alone used to mean "on", so @c VLA_EVO1_FA=0 turned flash attention + * *on* -- the opposite of what anyone typing it intends, and a quiet way to + * publish a benchmark of the wrong configuration. + * + * @c 0, @c false, @c off, @c no and the empty string are false (case + * insensitive); any other value is true; unset returns @p def. This matches + * @c VLA_GR00T_GRAPH_CACHE, which already parsed its value rather than its + * presence. + * + * @param name Variable to read. + * @param def Value to use when the variable is not set at all. + */ +inline bool env_flag(const char * name, bool def = false) { + const char * v = std::getenv(name); + if (!v) return def; + if (!*v) return false; // FOO= reads as "unset it" + + char buf[8] = {}; + size_t n = 0; + for (; n < sizeof(buf) - 1 && v[n]; ++n) { + buf[n] = (char) std::tolower((unsigned char) v[n]); + } + if (v[n]) return true; // longer than any false word, so it is one + + return !(std::strcmp(buf, "0") == 0 || + std::strcmp(buf, "false") == 0 || + std::strcmp(buf, "off") == 0 || + std::strcmp(buf, "no") == 0); +} + +} // namespace vla diff --git a/src/kernels/bitvla/bitnet_kernels.cu b/src/kernels/bitvla/bitnet_kernels.cu index 5669cd5..fe282b6 100644 --- a/src/kernels/bitvla/bitnet_kernels.cu +++ b/src/kernels/bitvla/bitnet_kernels.cu @@ -14,6 +14,8 @@ #include "bitnet_kernels.h" +#include "env_flag.h" + #include #include @@ -67,7 +69,7 @@ extern "C" void bitlinear_int8xint2(int8_t* input0, int8_t* input1, __nv_bfloat1 // VLA_BITVLA_NARROW_GEMM=1 to fall back to the one-tile-per-CTA kernel, which // is what the A/B correctness harness and any regression bisect want. static bool bitlinear_use_wide() { - static const bool wide = (std::getenv("VLA_BITVLA_NARROW_GEMM") == nullptr); + static const bool wide = !vla::env_flag("VLA_BITVLA_NARROW_GEMM"); return wide; } diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index b8a99e2..b908a6d 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -29,6 +29,7 @@ #include "kernels/bitvla/bitvla_lm_cuda.h" #include "kernels/bitvla/bitvla_vit_cuda.h" #include "kernels/bitvla/bitvla_fp32head_cuda.h" +#include "env_flag.h" #ifdef __GLIBC__ # include #endif @@ -536,7 +537,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_BITVLA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_BITVLA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; gguf_reader g("bitvla"); if (!g.open(ckpt_path)) return nullptr; @@ -676,7 +677,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, #ifdef VLA_BITVLA_CUDA_KERNELS - if ((m->packed_int2 || m->matmul_type == GGML_TYPE_F32) && !std::getenv("VLA_BITVLA_NO_CUDA_LM")) { + if ((m->packed_int2 || m->matmul_type == GGML_TYPE_F32) && !vla::env_flag("VLA_BITVLA_NO_CUDA_LM")) { int dev_count = 0; if (cudaGetDeviceCount(&dev_count) == cudaSuccess && dev_count > 0) { cudaSetDevice(0); @@ -881,7 +882,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, } if (m->cuda_lm_ready && m->cuda_vit_ready && - std::getenv("VLA_BITVLA_CPU_HEAD") == nullptr) { + !vla::env_flag("VLA_BITVLA_CPU_HEAD")) { m->fp32head_cuda_ctx = bitvla_fp32head_cuda_init( (int) m->proprio_dim, (int) m->lm_hidden, (int) m->num_actions_chunk, (int) m->action_dim, @@ -914,7 +915,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, } if (m->cuda_lm_ready && m->cuda_vit_ready && m->weight_buf && - std::getenv("VLA_BITVLA_KEEP_CPU_WEIGHTS") == nullptr) { + !vla::env_flag("VLA_BITVLA_KEEP_CPU_WEIGHTS")) { size_t bytes_kept = 0; if (!m->cuda_fp32head_ready) { diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 8807e31..7a16e42 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -24,6 +24,7 @@ #include "models/scratch_ctx.h" #include "models/act_dtype.h" #include "cuda/vla_cuda_ops.h" +#include "env_flag.h" #include #include @@ -199,7 +200,7 @@ bool preprocess_image_chw(const ImageView & v, int64_t side, std::vector // concentrated in the two tasks the control aced, so the default stays on the // accuracy-preserving path and the speedup is opt-in. inline bool evo1_vit_fa_enabled() { - static const bool enabled = (std::getenv("VLA_EVO1_FA") != nullptr); + static const bool enabled = vla::env_flag("VLA_EVO1_FA"); return enabled; } @@ -354,7 +355,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_EVO1_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = vla::env_flag("VLA_EVO1_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; @@ -375,7 +376,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. - if (std::getenv("VLA_EVO1_BF16_ACT")) { + if (vla::env_flag("VLA_EVO1_BF16_ACT")) { if (b.is_cuda && m->matmul_type == GGML_TYPE_BF16) { m->act_type = GGML_TYPE_BF16; cuda_register_bf16_ops(); // installs the in-tree BF16 CUDA kernels diff --git a/src/models/gr00tn1d5.cpp b/src/models/gr00tn1d5.cpp index 45c946f..3b2dfc9 100644 --- a/src/models/gr00tn1d5.cpp +++ b/src/models/gr00tn1d5.cpp @@ -24,6 +24,7 @@ #include "models/scratch_ctx.h" #include "models/vision_common.h" #include "models/dit_common.h" +#include "env_flag.h" #include #include @@ -259,7 +260,7 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index aff2f29..fdafdaa 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -23,6 +23,7 @@ #include "models/gguf_reader.h" #include "models/scratch_ctx.h" #include "models/dit_common.h" +#include "env_flag.h" #include #include @@ -299,7 +300,7 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index d9742a7..227399d 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -24,6 +24,7 @@ #include "models/scratch_ctx.h" #include "models/dit_common.h" #include "models/qwen3vl_vit.h" +#include "env_flag.h" #include #include @@ -311,7 +312,7 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; gguf_reader g("gr00tn1d7"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index 4650272..13e056e 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -24,6 +24,7 @@ #include "gguf.h" #include "models/gguf_reader.h" #include "models/scratch_ctx.h" +#include "env_flag.h" #include #include @@ -134,7 +135,7 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path if (!mmproj_path.empty()) std::printf("vla(openvla_oft): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = std::getenv("VLA_OPENVLA_OFT_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->mt = vla::env_flag("VLA_OPENVLA_OFT_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; gguf_reader g("openvla_oft"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 51f2f28..1396de3 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -27,6 +27,7 @@ #include "models/vision_common.h" #include "models/act_dtype.h" #include "cuda/vla_cuda_ops.h" +#include "env_flag.h" #include #include @@ -144,7 +145,7 @@ namespace { // cost. (The evo1 SR drop this used to cite did not reproduce.) // VLA_PI0_BF16_ACT is the better lever here: 9.1%, and its SR was measured. static inline bool pi0_fa_enabled() { - static const bool enabled = (std::getenv("VLA_PI0_FA") != nullptr); + static const bool enabled = vla::env_flag("VLA_PI0_FA"); return enabled; } @@ -362,7 +363,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = std::getenv("VLA_PI0_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = vla::env_flag("VLA_PI0_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; @@ -389,7 +390,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. - if (std::getenv("VLA_PI0_BF16_ACT")) { + if (vla::env_flag("VLA_PI0_BF16_ACT")) { if (b.is_cuda && m->matmul_type == GGML_TYPE_BF16) { m->act_type = GGML_TYPE_BF16; cuda_register_bf16_ops(); // installs the in-tree BF16 CUDA kernels diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 7e8c877..d51eee0 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -25,6 +25,7 @@ #include "models/scratch_ctx.h" #include "models/dit_common.h" #include "models/vision_common.h" +#include "env_flag.h" #include #include @@ -397,7 +398,7 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = std::getenv("VLA_PI05_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = vla::env_flag("VLA_PI05_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; @@ -742,7 +743,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { std::vector out((size_t) chunk * max_ad); ggml_backend_tensor_get(x_final, out.data(), 0, out.size() * sizeof(float)); - if (!std::getenv("VLA_PI05_SKIP_UNNORM")) { + if (!vla::env_flag("VLA_PI05_SKIP_UNNORM")) { for (int64_t t = 0; t < chunk; ++t) { float * row = out.data() + (size_t) t * max_ad; for (int64_t j = 0; j < max_ad; ++j) { diff --git a/src/models/qwen3vl_vit.h b/src/models/qwen3vl_vit.h index 9abc09a..b53cbb5 100644 --- a/src/models/qwen3vl_vit.h +++ b/src/models/qwen3vl_vit.h @@ -19,6 +19,7 @@ #include "model.h" #include "ggml.h" +#include "env_flag.h" #include #include @@ -43,7 +44,7 @@ inline ggml_tensor * rope2d(ggml_context * C, ggml_tensor * x, ggml_tensor * cos return ggml_add(C, ggml_mul(C, x, cos_t), ggml_mul(C, rot, sin_t)); } -inline bool fa_enabled() { static const bool e = (std::getenv("VLA_FLASH_ATTN") != nullptr); return e; } +inline bool fa_enabled() { static const bool e = vla::env_flag("VLA_FLASH_ATTN"); return e; } inline ggml_tensor * flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * mask, float scale) { diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 39ad167..a921d74 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -28,6 +28,7 @@ #include "backend.h" #include "nlohmann/json.hpp" +#include "env_flag.h" #include #include @@ -363,7 +364,7 @@ namespace { // 96/100 for explicit attention. evo1 showed the same ~4-5 pp drop, so the // default stays on the accuracy-preserving path. static inline bool siglip_fa_enabled() { - static const bool enabled = (std::getenv("VLA_SMOLVLA_FA") != nullptr); + static const bool enabled = vla::env_flag("VLA_SMOLVLA_FA"); return enabled; } diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 9351784..2f905a4 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -25,6 +25,7 @@ #include "models/gguf_reader.h" #include "models/scratch_ctx.h" #include "models/dit_common.h" +#include "env_flag.h" #include #include @@ -158,7 +159,7 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path if (!mmproj_path.empty()) std::printf("vla(vla_adapter): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = std::getenv("VLA_ADAPTER_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->mt = vla::env_flag("VLA_ADAPTER_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; gguf_reader g("vla_adapter"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 91549cf..2f32eef 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -24,6 +24,7 @@ #include "models/scratch_ctx.h" #include "models/dit_common.h" #include "models/qwen3vl_vit.h" +#include "env_flag.h" #include #include @@ -238,7 +239,7 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = std::getenv("VLA_JEPA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_JEPA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; gguf_reader g("vla_jepa"); if (!g.open(ckpt_path)) return nullptr; From 4d29e95ec0e82d75db3a52548de59b9166e2a479 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 17:13:31 +0700 Subject: [PATCH 05/21] widen the bf16 elementwise path to eight elements per thread on contiguous rows --- src/cuda/vla_cuda_bf16.cu | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index 8390a83..6afc426 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -120,6 +120,50 @@ __global__ void k_bin_bcast_bf16_rows( } } +// Vectorized row kernel: 8 BF16 per thread (one uint4 load/store), for the +// shape that dominates -- dst/src0 contiguous along dim 0 and src1 a full-width +// row repeated over the other dims, i.e. every bias add and norm-weight mul. +// The arithmetic is per element in float exactly as the scalar path does it, so +// results are bit-identical; only the memory access widens. Worth doing because +// a graph-node-level profile puts these kernels at ~27 ms of evo1's 147, and +// their mean/median split (6.3 vs 2.2 us) says the big tensors carry the total. +template +__global__ void k_bin_bcast_bf16_vec8( + const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, + __nv_bfloat16 * __restrict__ dst, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t s11, const int64_t s12, const int64_t s13, + const int64_t d1, const int64_t d2, const int64_t d3) { + const int64_t i1 = blockIdx.y; + const int64_t i23 = blockIdx.z; + const int64_t i2 = i23 % ne2; + const int64_t i3 = i23 / ne2; + + const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; + const S1 * __restrict__ r1 = src1 + bcast_idx(i1, ne11, ne1)*s11 + + bcast_idx(i2, ne12, ne2)*s12 + + bcast_idx(i3, ne13, ne3)*s13; + __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; + + const int64_t nvec = ne0 / 8; + for (int64_t v = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; v < nvec; + v += (int64_t) gridDim.x*blockDim.x) { + const int64_t i0 = v * 8; + + uint4 a = *reinterpret_cast(r0 + i0); + __nv_bfloat16 * av = reinterpret_cast<__nv_bfloat16 *>(&a); + + #pragma unroll + for (int k = 0; k < 8; ++k) { + const float b = (float) r1[i0 + k]; + av[k] = f2bf(apply_bin(op, bf2f(av[k]), b)); + } + *reinterpret_cast(rd + i0) = a; + } +} + // Fallback for shapes the row grid cannot address (gridDim.y/z cap at 65535). template __global__ void k_bin_bcast_bf16_flat( @@ -196,6 +240,41 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { const RowGrid g = force_flat() ? RowGrid{} : row_grid(dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + // 8-wide path: dim 0 contiguous on both sides and src1 a full-width row. + // Requires 16 B alignment for the uint4 accesses, which the ggml allocator + // gives at tensor start but not necessarily at a strided row offset. + const bool vec8_shape = + g.ok && dst->ne[0] % 8 == 0 && + es(src0, 0) == 1 && es(dst, 0) == 1 && es(src1, 0) == 1 && + src1->ne[0] == dst->ne[0] && + es(src0, 1) % 8 == 0 && es(dst, 1) % 8 == 0 && es(src1, 1) % 8 == 0 && + ((uintptr_t) src0->data % 16) == 0 && ((uintptr_t) dst->data % 16) == 0; + if (vec8_shape) { + const int64_t nvec = dst->ne[0] / 8; + unsigned bx = 32; + while (bx < (unsigned) BLOCK && (int64_t) bx < nvec) bx *= 2; + int64_t gx = (nvec + bx - 1) / bx; + if (gx > 65535) gx = 65535; + if (gx < 1) gx = 1; + const dim3 vgrid((unsigned) gx, g.grid.y, g.grid.z); + const dim3 vblock(bx, 1, 1); + +#define VLA_LAUNCH_VEC8(TYPE) \ + k_bin_bcast_bf16_vec8<<>>( \ + (const __nv_bfloat16 *) src0->data, (const TYPE *) src1->data, \ + (__nv_bfloat16 *) dst->data, \ + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], \ + es(src0,1), es(src0,2), es(src0,3), \ + src1->ne[1], src1->ne[2], src1->ne[3], \ + es(src1,1), es(src1,2), es(src1,3), \ + es(dst,1), es(dst,2), es(dst,3)) + + if (src1->type == GGML_TYPE_BF16) { VLA_LAUNCH_VEC8(__nv_bfloat16); } + else { VLA_LAUNCH_VEC8(float); } +#undef VLA_LAUNCH_VEC8 + return true; + } + #define VLA_LAUNCH_BIN(TYPE) \ do { \ if (g.ok) { \ From 87a7e8feab0dbd084886264ab3b5cb453d5f8487 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 21:36:29 +0700 Subject: [PATCH 06/21] split the model tree into layer, module and model levels --- CMakeLists.txt | 7 + eval/refactor_verify.sh | 83 ++ src/{models => }/act_dtype.h | 0 src/{models => }/gguf_reader.h | 0 src/layers/attn.h | 62 ++ src/layers/embed.h | 87 ++ src/layers/ffn.h | 46 ++ src/layers/linear.h | 48 ++ src/layers/norm.h | 50 ++ src/layers/rope.h | 88 +++ src/loader.cpp | 173 ++++ src/loader.h | 87 ++ src/models/bitvla.cpp | 4 +- src/models/evo1.cpp | 6 +- src/models/gr00tn1d5.cpp | 658 +++++++--------- src/models/gr00tn1d6.cpp | 744 ++++++++---------- src/models/gr00tn1d7.cpp | 6 +- src/models/openvla_oft.cpp | 8 +- src/models/pi0.cpp | 8 +- src/models/pi05.cpp | 6 +- src/models/smolvla.cpp | 4 +- src/models/vla_adapter.cpp | 8 +- src/models/vla_jepa.cpp | 6 +- src/modules/action_expert.cpp | 60 ++ src/modules/action_expert.h | 50 ++ src/modules/dit_head.cpp | 141 ++++ src/modules/dit_head.h | 76 ++ src/{models => modules}/dual_tower.h | 0 src/modules/encoder.cpp | 79 ++ src/modules/encoder.h | 66 ++ .../vision_common.h => modules/preprocess.h} | 45 ++ src/modules/prompt.cpp | 84 ++ src/modules/prompt.h | 52 ++ src/modules/qwen3_lm.cpp | 85 ++ src/modules/qwen3_lm.h | 62 ++ src/{models => modules}/qwen3vl_vit.h | 0 src/modules/siglip_vit.cpp | 51 ++ src/modules/siglip_vit.h | 49 ++ src/{models => }/scratch_ctx.h | 0 tests/bitvla_gemm_check.cu | 238 ++++++ tests/test_bf16_cuda_ops.cpp | 2 +- tests/test_qwen3vl_vit.cpp | 2 +- tests/test_vision_common.cpp | 2 +- 43 files changed, 2472 insertions(+), 861 deletions(-) create mode 100755 eval/refactor_verify.sh rename src/{models => }/act_dtype.h (100%) rename src/{models => }/gguf_reader.h (100%) create mode 100644 src/layers/attn.h create mode 100644 src/layers/embed.h create mode 100644 src/layers/ffn.h create mode 100644 src/layers/linear.h create mode 100644 src/layers/norm.h create mode 100644 src/layers/rope.h create mode 100644 src/loader.cpp create mode 100644 src/loader.h create mode 100644 src/modules/action_expert.cpp create mode 100644 src/modules/action_expert.h create mode 100644 src/modules/dit_head.cpp create mode 100644 src/modules/dit_head.h rename src/{models => modules}/dual_tower.h (100%) create mode 100644 src/modules/encoder.cpp create mode 100644 src/modules/encoder.h rename src/{models/vision_common.h => modules/preprocess.h} (60%) create mode 100644 src/modules/prompt.cpp create mode 100644 src/modules/prompt.h create mode 100644 src/modules/qwen3_lm.cpp create mode 100644 src/modules/qwen3_lm.h rename src/{models => modules}/qwen3vl_vit.h (100%) create mode 100644 src/modules/siglip_vit.cpp create mode 100644 src/modules/siglip_vit.h rename src/{models => }/scratch_ctx.h (100%) create mode 100644 tests/bitvla_gemm_check.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index 8119a6c..1b4992d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,13 @@ vla_exclude_fetched_targets(${llama_SOURCE_DIR}) add_library(vla_core src/model.cpp + src/loader.cpp + src/modules/action_expert.cpp + src/modules/dit_head.cpp + src/modules/encoder.cpp + src/modules/prompt.cpp + src/modules/qwen3_lm.cpp + src/modules/siglip_vit.cpp src/models/smolvla.cpp src/models/pi0.cpp src/models/pi05.cpp diff --git a/eval/refactor_verify.sh b/eval/refactor_verify.sh new file mode 100755 index 0000000..4b37e1c --- /dev/null +++ b/eval/refactor_verify.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright 2026 VinRobotics - Apache-2.0 +# +# Bit-exactness harness for the src/models -> layer/module/model refactor. +# +# Runs tests/predict_check over every arch's real GGUF with fixed images / +# language / state / noise and writes one action-chunk file per arch. Capture a +# baseline before touching the code, then re-run after each step and diff: +# +# eval/refactor_verify.sh outputs/refactor/base +# ...refactor... +# eval/refactor_verify.sh outputs/refactor/new +# diff -r outputs/refactor/base outputs/refactor/new && echo BIT-EXACT +# +# ARCHS=... restricts the sweep to a subset (space separated, names below). +# The square input side is probed rather than hardcoded: predict_check defaults +# to 224 and an arch whose tower wants another side returns action_len=0 on a +# mismatch instead of failing, so a wrong side would silently "pass" a diff. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +BIN="${BIN:-${REPO_ROOT}/build/tests/vla_predict_check}" +HF="${HF:-/mnt/data/hf_data/vrfai}" +OUT="${1:-${REPO_ROOT}/outputs/refactor/baseline}" +SIDES="${SIDES:-224 256 448 512}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +# arch|ckpt|mmproj|n_images|extra env +MODELS=( + "smolvla|${HF}/smolvla-libero-gguf/smolvla-libero.gguf|${HF}/backup/mmproj-smolvla-libero.gguf|2|" + "pi0|${HF}/pi0-libero-finetuned-v044-gguf/pi0-libero-finetuned-v044.gguf|${HF}/backup/mmproj-pi0-libero-finetuned-v044.gguf|2|" + "pi05|${HF}/pi05-libero-gguf/pi05-libero.gguf|${HF}/backup/mmproj-pi05-libero.gguf|2|" + "evo1|${HF}/evo1-libero-gguf/evo1-libero.gguf||2|" + "gr00t_n1_5|${HF}/gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf||2|" + "gr00t_n1_6|${HF}/gr00tn1d6-libero-gguf/gr00tn1d6-libero.gguf||2|" + "gr00t_n1_7|${HF}/gr00tn1d7-libero-gguf/libero_object/gr00tn1d7-libero-object.gguf||2|" + "bitvla|${HF}/bitvla-libero-gguf/libero_object/bitvla-libero-object.gguf||2|" + "vla_adapter|${HF}/vla-adapter-libero-object-gguf/libero_object/vla-adapter-libero-object.gguf||2|" + "openvla_oft|${HF}/openvla-oft-libero-gguf/openvla-oft-libero.gguf||2|" + "vla_jepa|${HF}/vla-jepa-libero/vla-jepa.gguf||2|VLA_EXTRA_TOKEN=151697 VLA_EXTRA_COUNT=32" +) + +[[ -x "${BIN}" ]] || { echo "ERROR: missing ${BIN} (cmake -DVLA_BUILD_TESTS=ON)" >&2; exit 1; } +mkdir -p "${OUT}" + +fail=0 +for row in "${MODELS[@]}"; do + IFS='|' read -r arch ckpt mmproj nimg extra <<< "${row}" + + if [[ -n "${ARCHS:-}" && " ${ARCHS} " != *" ${arch} "* ]]; then + continue + fi + if [[ ! -e "${ckpt}" ]]; then + echo "[skip] ${arch}: no checkpoint at ${ckpt}" + continue + fi + + ok=0 + for side in ${SIDES}; do + # shellcheck disable=SC2086 + if env ${extra} VLA_IMG_SIZE="${side}" "${BIN}" "${ckpt}" "${mmproj}" "${nimg}" \ + > "${OUT}/${arch}.actions.txt" 2> "${OUT}/${arch}.log"; then + if ! grep -q '^action_len=0$' "${OUT}/${arch}.actions.txt"; then + echo "[ok ] ${arch} side=${side} $(head -1 "${OUT}/${arch}.actions.txt")" + echo "${side}" > "${OUT}/${arch}.side" + ok=1 + break + fi + fi + done + + if [[ "${ok}" -eq 0 ]]; then + echo "[FAIL] ${arch}: no input side in '${SIDES}' produced a chunk; see ${OUT}/${arch}.log" >&2 + fail=1 + fi +done + +echo +echo "actions written to ${OUT}" +exit "${fail}" diff --git a/src/models/act_dtype.h b/src/act_dtype.h similarity index 100% rename from src/models/act_dtype.h rename to src/act_dtype.h diff --git a/src/models/gguf_reader.h b/src/gguf_reader.h similarity index 100% rename from src/models/gguf_reader.h rename to src/gguf_reader.h diff --git a/src/layers/attn.h b/src/layers/attn.h new file mode 100644 index 0000000..70c73a8 --- /dev/null +++ b/src/layers/attn.h @@ -0,0 +1,62 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Scaled dot-product attention. `nv` is the batch (view) count; ggml pads every +// shape to four dimensions, so nv == 1 emits the same nodes as the 2d/3d +// spelling it replaces. + +#pragma once + +#include "ggml.h" + +#include + +namespace vla { + +// [heads*hd, T, nv] -> [hd, T, heads, nv]. +inline ggml_tensor * to_heads(ggml_context * C, ggml_tensor * p, int64_t hd, int64_t heads, + int64_t T, int64_t nv = 1) { + return ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, p, hd, heads, T, nv), 0, 2, 1, 3)); +} + +// V is pre-transposed so that mul_mat(V, aw) lands the right way round. +inline ggml_tensor * to_heads_v(ggml_context * C, ggml_tensor * p, int64_t hd, int64_t heads, + int64_t T, int64_t nv = 1) { + return ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, p, hd, heads, T, nv), 1, 2, 0, 3)); +} + +// Scores stay F32 whatever the activation dtype: softmax over a BF16 reduction +// loses too much. +inline ggml_tensor * attention(ggml_context * C, ggml_tensor * Q, ggml_tensor * K, ggml_tensor * V, + ggml_tensor * mask, float scale, int64_t dim, int64_t T, int64_t nv = 1) { + ggml_tensor * kq = ggml_mul_mat(C, K, Q); + ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + + ggml_tensor * aw = ggml_soft_max_ext(C, kq, mask, scale, 0.0f); + ggml_tensor * kqv = ggml_mul_mat(C, V, aw); + return ggml_reshape_3d(C, ggml_cont(C, ggml_permute(C, kqv, 0, 2, 1, 3)), dim, T, nv); +} + +// Takes V laid out like Q/K, not the transposed to_heads_v form. +inline ggml_tensor * flash_attention(ggml_context * C, ggml_tensor * Q, ggml_tensor * K, ggml_tensor * V, + ggml_tensor * mask, float scale) { + ggml_tensor * kf = K->type == GGML_TYPE_F16 ? K : ggml_cast(C, K, GGML_TYPE_F16); + ggml_tensor * vf = V->type == GGML_TYPE_F16 ? V : ggml_cast(C, V, GGML_TYPE_F16); + + ggml_tensor * o = ggml_flash_attn_ext(C, Q, kf, vf, mask, scale, 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32); + return ggml_reshape_2d(C, o, o->ne[0]*o->ne[1], o->ne[2]*o->ne[3]); +} + +} diff --git a/src/layers/embed.h b/src/layers/embed.h new file mode 100644 index 0000000..030c0d1 --- /dev/null +++ b/src/layers/embed.h @@ -0,0 +1,87 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Host-side input tables, uploaded as graph inputs rather than built as nodes. +// The sin/cos order differs per family and each matches its reference; +// tests/test_dit_common.cpp pins all three. + +#pragma once + +#include +#include +#include +#include + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +namespace vla { + +// cos first, then sin. +inline void timesteps_proj(int64_t bucket, std::vector & out) { + const int64_t half = 128; + const float lm = std::log(10000.0f); + const float t = (float)bucket; + + out.assign(256, 0.0f); + for (int64_t i=0; i & out) { + const int64_t half = dim/2; + const float step = std::log(10000.0f)/(float)half; + const float t = (float)bucket; + + out.assign((size_t)T*dim, 0.0f); + for (int64_t tk=0; tk sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { + const int64_t half = dim/2; + + std::vector out(dim); + for (int64_t i=0; i & out) { + const float NEG = -std::numeric_limits::infinity(); + + out.assign((size_t)seq*seq, 0.0f); + for (int64_t q=0; q + +namespace vla { + +inline ggml_tensor * linear(ggml_context * C, ggml_tensor * W, ggml_tensor * b, ggml_tensor * x) { + ggml_tensor * y = ggml_mul_mat(C, W, x); + return b ? ggml_add(C, y, b) : y; +} + +// One row of a stacked [out, in, n_embodiment] weight: the GR00T action expert +// keeps a per-embodiment copy of every projection in one tensor. +inline ggml_tensor * cat_linear(ggml_context * C, ggml_tensor * W3d, ggml_tensor * b2d, int64_t id, ggml_tensor * x) { + const int64_t out = W3d->ne[0]; + const int64_t in = W3d->ne[1]; + + ggml_tensor * W_id = ggml_view_2d(C, W3d, out, in, W3d->nb[1], (size_t)id*W3d->nb[2]); + ggml_tensor * y = ggml_mul_mat(C, ggml_cont(C, ggml_transpose(C, W_id)), x); + return ggml_add(C, y, ggml_view_1d(C, b2d, out, (size_t)id*b2d->nb[1])); +} + +// Slice block `blk` out of a fused [nblk*E, T] projection, laid out as heads. +inline ggml_tensor * head_view(ggml_context * C, ggml_tensor * proj, int64_t hd, int64_t heads, + int64_t T, int64_t E, int nblk, int blk) { + const size_t es = ggml_element_size(proj); + return ggml_view_3d(C, proj, hd, heads, T, (size_t)hd*es, (size_t)nblk*E*es, (size_t)blk*E*es); +} + +} diff --git a/src/layers/norm.h b/src/layers/norm.h new file mode 100644 index 0000000..93cd3e5 --- /dev/null +++ b/src/layers/norm.h @@ -0,0 +1,50 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Normalisation layers. + +#pragma once + +#include "layers/linear.h" + +#include "ggml.h" + +#include +#include + +namespace vla { + +inline ggml_tensor * layer_norm(ggml_context * C, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, float eps) { + return ggml_add(C, ggml_mul(C, ggml_norm(C, x, eps), w), b); +} + +// w == nullptr leaves the norm unscaled. +inline ggml_tensor * rms_norm(ggml_context * C, ggml_tensor * x, ggml_tensor * w, float eps) { + ggml_tensor * n = ggml_rms_norm(C, x, eps); + return w ? ggml_mul(C, n, w) : n; +} + +// The conditioning vector is (scale, shift) in that order; the final projection +// layer of each DiT head uses (shift, scale) instead. +inline ggml_tensor * adaln(ggml_context * C, ggml_tensor * x, ggml_tensor * temb, + ggml_tensor * lw, ggml_tensor * lb, int64_t dim, float eps) { + ggml_tensor * cond = linear(C, lw, lb, ggml_silu(C, temb)); + ggml_tensor * sc = ggml_view_1d(C, cond, dim, 0); + ggml_tensor * sh = ggml_view_1d(C, cond, dim, (size_t)dim*sizeof(float)); + + ggml_tensor * xn = ggml_norm(C, x, eps); + return ggml_add(C, ggml_add(C, xn, ggml_mul(C, xn, sc)), sh); +} + +} diff --git a/src/layers/rope.h b/src/layers/rope.h new file mode 100644 index 0000000..8874bbd --- /dev/null +++ b/src/layers/rope.h @@ -0,0 +1,88 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Rotary embeddings. Three conventions live in the tree and they are not +// interchangeable; the pairing is pinned by tests/test_rope_conventions.cpp. +// +// RopeSpec ggml rope, NEOX or IMROPE. The Qwen3 backbones. +// rope_2d precomputed tables, half-split rotation. The Qwen3-VL tower, +// whose 2d grid positions ggml_rope has no spelling for. +// rope_pairwise precomputed tables, adjacent-pair rotation. VLA-Adapter's +// action head pairs a half-split frequency table with an +// interleaved rotation; the reference does the same, so the +// mismatch is deliberate. + +#pragma once + +#include "ggml.h" + +#include + +namespace vla { + +// Which ggml rope call a backbone wants, and with what parameters. `sections` +// is read only when type is GGML_ROPE_TYPE_IMROPE. +struct RopeSpec { + int type = GGML_ROPE_TYPE_NEOX; + int n_dims = 0; + int sections[4]= {0, 0, 0, 0}; + float freq_base = 10000.0f; + float freq_scale = 1.0f; + float ext_factor = 0.0f; + float attn_factor= 1.0f; + float beta_fast = 32.0f; + float beta_slow = 1.0f; +}; + +inline ggml_tensor * rope(ggml_context * C, const RopeSpec & r, ggml_tensor * x, ggml_tensor * pos) { + if (r.type == GGML_ROPE_TYPE_IMROPE) { + int sect[4] = { r.sections[0], r.sections[1], r.sections[2], r.sections[3] }; + return ggml_rope_multi(C, x, pos, nullptr, r.n_dims, sect, r.type, 0, + r.freq_base, r.freq_scale, r.ext_factor, r.attn_factor, r.beta_fast, r.beta_slow); + } + return ggml_rope_ext(C, x, pos, nullptr, r.n_dims, r.type, 0, + r.freq_base, r.freq_scale, r.ext_factor, r.attn_factor, r.beta_fast, r.beta_slow); +} + +// Half-split rotation against precomputed tables: (x1, x2) -> (-x2, x1). +inline ggml_tensor * rope_2d(ggml_context * C, ggml_tensor * x, ggml_tensor * cos_t, ggml_tensor * sin_t) { + const int64_t hd = x->ne[0]; + const int64_t S = x->ne[1]; + const int64_t Hh = x->ne[2]; + const int64_t half = hd/2; + + ggml_tensor * x1 = ggml_cont(C, ggml_view_3d(C, x, half, S, Hh, x->nb[1], x->nb[2], 0)); + ggml_tensor * x2 = ggml_cont(C, ggml_view_3d(C, x, half, S, Hh, x->nb[1], x->nb[2], (size_t)half*x->nb[0])); + ggml_tensor * rot = ggml_concat(C, ggml_neg(C, x2), x1, 0); + return ggml_add(C, ggml_mul(C, x, cos_t), ggml_mul(C, rot, sin_t)); +} + +// Adjacent-pair rotation: (even, odd) -> (-odd, even). +inline ggml_tensor * rope_pairwise_rot(ggml_context * C, ggml_tensor * x, int64_t HD) { + const int64_t L = x->ne[1]; + const int64_t H = x->ne[2]; + + ggml_tensor * xp = ggml_reshape_4d(C, x, 2, HD/2, L, H); + ggml_tensor * ev = ggml_cont(C, ggml_view_4d(C, xp, 1, HD/2, L, H, xp->nb[1], xp->nb[2], xp->nb[3], 0)); + ggml_tensor * od = ggml_cont(C, ggml_view_4d(C, xp, 1, HD/2, L, H, xp->nb[1], xp->nb[2], xp->nb[3], xp->nb[0])); + return ggml_reshape_3d(C, ggml_concat(C, ggml_scale(C, od, -1.0f), ev, 0), HD, L, H); +} + +inline ggml_tensor * rope_pairwise(ggml_context * C, ggml_tensor * x, ggml_tensor * cs, ggml_tensor * sn, int64_t HD) { + ggml_tensor * c = ggml_reshape_3d(C, cs, HD, x->ne[1], 1); + ggml_tensor * s = ggml_reshape_3d(C, sn, HD, x->ne[1], 1); + return ggml_add(C, ggml_mul(C, x, c), ggml_mul(C, rope_pairwise_rot(C, x, HD), s)); +} + +} diff --git a/src/loader.cpp b/src/loader.cpp new file mode 100644 index 0000000..ab7e0ae --- /dev/null +++ b/src/loader.cpp @@ -0,0 +1,173 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "loader.h" + +#include +#include +#include + +namespace vla { + +namespace { + +// GGUF tensor names in this tree top out well under this; the truncation guard +// below turns an overflow into a load failure rather than a silent wrong name. +constexpr size_t NAME_CAP = 256; + +} + +ggml_tensor * WeightLoader::declare(ggml_type want, bool required, bool gemma_norm, + const char * fmt, va_list ap) { + char name[NAME_CAP]; + const int n = std::vsnprintf(name, sizeof(name), fmt, ap); + if (n < 0 || (size_t)n >= sizeof(name)) { + std::fprintf(stderr, "vla(%s): tensor name too long for a %zu-byte buffer\n", arch_, sizeof(name)); + ok_ = false; + return nullptr; + } + + const ggml_tensor * src = g_.meta(name); + if (!src) { + if (required) { + std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch_, name); + ok_ = false; + } + return nullptr; + } + + ggml_tensor * t = ggml_new_tensor(ctx_, g_.resident_type(src, want), ggml_n_dims(src), src->ne); + if (!t) { + std::fprintf(stderr, "vla(%s): ggml_new_tensor failed for %s (weight context too small?)\n", arch_, name); + ok_ = false; + return nullptr; + } + ggml_set_name(t, name); + + if (gemma_norm) gemma_norms_.push_back(name); + return t; +} + +// The five entry points differ only in the resident type and whether a miss is +// fatal, so each is a one-line forward into declare(). +#define VLA_DECLARE_FN(fn, type, required, gemma) \ + ggml_tensor * WeightLoader::fn(const char * fmt, ...) { \ + va_list ap; \ + va_start(ap, fmt); \ + ggml_tensor * t = declare(type, required, gemma, fmt, ap); \ + va_end(ap); \ + return t; \ + } + +VLA_DECLARE_FN(gemm, gemm_, true, false) +VLA_DECLARE_FN(f32, GGML_TYPE_F32, true, false) +VLA_DECLARE_FN(opt_gemm, gemm_, false, false) +VLA_DECLARE_FN(opt_f32, GGML_TYPE_F32, false, false) +VLA_DECLARE_FN(f32_gemma_norm, GGML_TYPE_F32, true, true) + +#undef VLA_DECLARE_FN + +ggml_tensor * WeightLoader::fuse_gemm(const char * out_name, const std::vector & srcs) { + return fuse(gemm_, out_name, srcs); +} + +ggml_tensor * WeightLoader::fuse_f32(const char * out_name, const std::vector & srcs) { + return fuse(GGML_TYPE_F32, out_name, srcs); +} + +ggml_tensor * WeightLoader::fuse(ggml_type want, const char * out_name, const std::vector & srcs) { + if (srcs.empty()) { ok_ = false; return nullptr; } + + const ggml_tensor * first = g_.meta(srcs[0].c_str()); + if (!first) { + std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch_, srcs[0].c_str()); + ok_ = false; + return nullptr; + } + + const bool is1d = ggml_n_dims(first) == 1; + int64_t rows = 0; + for (const std::string & s : srcs) { + const ggml_tensor * gs = g_.meta(s.c_str()); + if (!gs) { + std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch_, s.c_str()); + ok_ = false; + return nullptr; + } + rows += is1d ? gs->ne[0] : gs->ne[1]; + } + + ggml_tensor * t = is1d ? ggml_new_tensor_1d(ctx_, want, rows) + : ggml_new_tensor_2d(ctx_, want, first->ne[0], rows); + if (!t) { + std::fprintf(stderr, "vla(%s): ggml_new_tensor failed for %s\n", arch_, out_name); + ok_ = false; + return nullptr; + } + ggml_set_name(t, out_name); + fused_.push_back(Fused{t, srcs}); + return t; +} + +bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_buf) { + if (!ok_) { + std::fprintf(stderr, "vla(%s): weight tensor setup failed\n", arch_); + return false; + } + + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_, backend); + if (!buf) { + std::fprintf(stderr, "vla(%s): ggml_backend_alloc_ctx_tensors failed (OOM?)\n", arch_); + return false; + } + *out_buf = buf; + + for (ggml_tensor * t = ggml_get_first_tensor(ctx_); t; t = ggml_get_next_tensor(ctx_, t)) { + const char * name = ggml_get_name(t); + const bool fused = std::any_of(fused_.begin(), fused_.end(), + [&](const Fused & f) { return f.dst == t; }); + if (fused) continue; + + const bool gn = std::find(gemma_norms_.begin(), gemma_norms_.end(), name) != gemma_norms_.end(); + + std::vector bytes = g_.read_convert(name, t->type, gn); + if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { + std::fprintf(stderr, "vla(%s): failed to load %s (%zu vs %zu bytes)\n", + arch_, name, bytes.size(), ggml_nbytes(t)); + return false; + } + ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); + } + + for (const Fused & f : fused_) { + std::vector buf; + for (const std::string & s : f.srcs) { + std::vector b = g_.read_convert(s.c_str(), f.dst->type); + if (b.empty()) { + std::fprintf(stderr, "vla(%s): fused fill: read %s failed\n", arch_, s.c_str()); + return false; + } + buf.insert(buf.end(), b.begin(), b.end()); + } + if (buf.size() != ggml_nbytes(f.dst)) { + std::fprintf(stderr, "vla(%s): fused fill: %s size %zu vs %zu\n", + arch_, ggml_get_name(f.dst), buf.size(), ggml_nbytes(f.dst)); + return false; + } + ggml_backend_tensor_set(f.dst, buf.data(), 0, buf.size()); + } + return true; +} + +} diff --git a/src/loader.h b/src/loader.h new file mode 100644 index 0000000..3f52bcd --- /dev/null +++ b/src/loader.h @@ -0,0 +1,87 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Weight declaration and upload. A miss is recorded on the loader and surfaces +// once, at ok(), so a module's declare() can stay a flat list of names. +// +// gemm() lands in the model's matmul type, unless the GGUF holds the tensor +// quantized, in which case it stays packed and ggml dequantizes at compute. +// f32() is for norms, biases and embedding tables. + +#pragma once + +#include "gguf_reader.h" + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include + +namespace vla { + +class WeightLoader { +public: + WeightLoader(const char * arch, gguf_reader & g, ggml_context * ctx, ggml_type gemm_type) + : arch_(arch), g_(g), ctx_(ctx), gemm_(gemm_type) {} + + WeightLoader(const WeightLoader &) = delete; + WeightLoader & operator=(const WeightLoader &) = delete; + + // printf-formatted so a per-block prefix needs no scratch buffer at the + // call site. A miss returns nullptr and fails ok(). + ggml_tensor * gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); + ggml_tensor * f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); + + // A miss is not an error; the caller branches on nullptr. + ggml_tensor * opt_gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); + ggml_tensor * opt_f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); + + // Gemma norms are centred on zero and add 1 at use; folding the +1 in at + // load keeps it off the graph and needs unpacked floats. + ggml_tensor * f32_gemma_norm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); + + // One resident tensor holding several GGUF tensors concatenated along the + // outer dimension, so a split projection can be issued as a single GEMM. + // `out_name` is synthetic and need not exist in the file. + ggml_tensor * fuse_gemm(const char * out_name, const std::vector & srcs); + ggml_tensor * fuse_f32 (const char * out_name, const std::vector & srcs); + + ggml_type gemm_type() const { return gemm_; } + bool ok() const { return ok_; } + + // One backend buffer for everything declared so far, then fills it. + bool upload(ggml_backend_t backend, ggml_backend_buffer_t * out_buf); + +private: + ggml_tensor * declare(ggml_type want, bool required, bool gemma_norm, const char * fmt, va_list ap); + ggml_tensor * fuse(ggml_type want, const char * out_name, const std::vector & srcs); + + const char * arch_; + gguf_reader & g_; + ggml_context * ctx_; + ggml_type gemm_; + bool ok_ = true; + + struct Fused { + ggml_tensor * dst; + std::vector srcs; + }; + + std::vector gemma_norms_; + std::vector fused_; +}; + +} diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index b908a6d..c3f6a68 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -22,8 +22,8 @@ #include "ggml-cuda.h" #endif #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #ifdef VLA_BITVLA_CUDA_KERNELS #include "kernels/bitvla/bitvla_lm_cuda.h" diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 7a16e42..af35921 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -20,9 +20,9 @@ #include "ggml-backend.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" -#include "models/act_dtype.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" +#include "act_dtype.h" #include "cuda/vla_cuda_ops.h" #include "env_flag.h" diff --git a/src/models/gr00tn1d5.cpp b/src/models/gr00tn1d5.cpp index 3b2dfc9..21a50ae 100644 --- a/src/models/gr00tn1d5.cpp +++ b/src/models/gr00tn1d5.cpp @@ -12,53 +12,52 @@ // See the License for the specific language governing permissions and // limitations under the License. +// NVIDIA Isaac GR00T N1.5: SigLIP tower -> Qwen3 backbone -> VLSA encoder -> +// DiT action head under a flow-matching solver. + #include "arch.h" +#include "backend.h" +#include "env_flag.h" +#include "gguf_reader.h" +#include "layers/embed.h" +#include "layers/linear.h" +#include "layers/norm.h" #include "model.h" +#include "modules/action_expert.h" +#include "modules/dit_head.h" +#include "modules/encoder.h" +#include "modules/preprocess.h" +#include "modules/prompt.h" +#include "modules/qwen3_lm.h" +#include "modules/siglip_vit.h" +#include "scratch_ctx.h" #include "ggml.h" -#include "ggml-cpu.h" #include "ggml-backend.h" -#include "backend.h" -#include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" -#include "models/vision_common.h" -#include "models/dit_common.h" -#include "env_flag.h" #include -#include #include #include #include #include -#include #include #include #include #include namespace vla { -namespace { - - -struct SigLipLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; -struct Qwen3LayerW { ggml_tensor *attn_norm,*Wq,*Wk,*Wv,*Wo,*q_norm,*k_norm,*ffn_norm,*Wgate,*Wup,*Wdown; }; -struct VlsaLayerW { ggml_tensor *n1w,*n1b,*n3w,*n3b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; }; -struct DitLayerW { ggml_tensor *adaln_w,*adaln_b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; }; - -} struct Gr00tN1d5ModelArch : public ModelArchBase { Gr00tN1d5ModelArch() : ModelArchBase(Arch::GR00T_N1_5) {} ~Gr00tN1d5ModelArch() override; - std::string gguf_path; // Opened once at load: reopening per predict re-parses the whole GGUF header. gguf_reader io{"gr00tn1d5"}; ggml_backend_t backend = nullptr; int n_threads = default_cpu_threads(); ggml_context * ctx_weights = nullptr; + ggml_backend_buffer_t weight_buf = nullptr; + ggml_type matmul_type = GGML_TYPE_F32; scratch_ctx vision_scratch; struct MainKey { @@ -70,177 +69,123 @@ struct Gr00tN1d5ModelArch : public ModelArchBase { std::vector t_tau, t_tproj; }; graph_cache main_graph; - ggml_backend_buffer_t weight_buf = nullptr; - ggml_type matmul_type = GGML_TYPE_F32; - int64_t vit_hidden=1152, vit_layers=27, vit_heads=16, vit_inter=4304, image_size=224, patch_size=14, n_img_tokens=256; - int64_t lm_hidden=2048, lm_layers=12, n_q=16, n_kv=8, lm_head_dim=128, lm_inter=6144, vocab=151680, image_token_index=151669; - int64_t bb_embed_dim=2048, in_embed_dim=1536, dit_hidden=1536, dit_heads=32, dit_head_dim=48, dit_layers=16, dit_interleave=1; - int64_t vlsa_layers=4, vlsa_heads=32, vlsa_head_dim=64, vlsa_inter=8192; + SigLipTower vit; + Qwen3LM lm; + EncStack vlsa; + ActionExpert aex; + DitHead dit; + ggml_tensor *mm_W=nullptr, *mm_b=nullptr; + ggml_tensor *vlln_w=nullptr, *vlln_b=nullptr; + ggml_tensor *future_tokens=nullptr; + + int64_t vit_layers=27, vit_inter=4304, image_size=224, patch_size=14, n_img_tokens=256; + int64_t lm_inter=6144, vocab=151680, image_token_index=151669; + int64_t bb_embed_dim=2048, in_embed_dim=1536, dit_interleave=1, vlsa_layers=4; int64_t num_future=32, action_horizon=16, action_dim=32, max_state_dim=64; int64_t num_steps=4, num_buckets=1000, max_embodiments=32, max_seq_len=1024; - float vit_ln_eps=1e-6f, lm_rms_eps=1e-6f, ln_eps=1e-5f, norm_out_eps=1e-6f, vlln_eps=1e-5f, lm_rope_base=1000000.0f; - int64_t embodiment_id = 24; - - ggml_tensor *vit_patch_w=nullptr,*vit_patch_b=nullptr,*vit_pos=nullptr,*vit_post_ln_w=nullptr,*vit_post_ln_b=nullptr; - std::vector vit; - ggml_tensor *mm_W=nullptr,*mm_b=nullptr; - - ggml_tensor *lm_output_norm=nullptr; - std::vector lm; - - ggml_tensor *vlln_w=nullptr,*vlln_b=nullptr; - std::vector vlsa; - ggml_tensor *se_l1W=nullptr,*se_l1b=nullptr,*se_l2W=nullptr,*se_l2b=nullptr; - ggml_tensor *ae_W1W=nullptr,*ae_W1b=nullptr,*ae_W2W=nullptr,*ae_W2b=nullptr,*ae_W3W=nullptr,*ae_W3b=nullptr; - ggml_tensor *ad_l1W=nullptr,*ad_l1b=nullptr,*ad_l2W=nullptr,*ad_l2b=nullptr; - ggml_tensor *future_tokens=nullptr,*pos_embd=nullptr; - ggml_tensor *te_l1W=nullptr,*te_l1b=nullptr,*te_l2W=nullptr,*te_l2b=nullptr; - std::vector dit; - ggml_tensor *po1W=nullptr,*po1b=nullptr,*po2W=nullptr,*po2b=nullptr; + float vlln_eps=1e-5f; std::vector predict(const Inputs& in) override; }; namespace { -ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, - int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) head_dim); - ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, n1), w.bv); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, head_dim, heads, seq), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq); - ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); - return ggml_add(C, h1, ff); -} - -ggml_tensor * build_qwen3_layer(ggml_context * C, const Gr00tN1d5ModelArch & m, const Qwen3LayerW & w, - ggml_tensor * h, ggml_tensor * positions, ggml_tensor * mask, int64_t seq) { - const int64_t hd = m.lm_head_dim, n_q = m.n_q, n_kv = m.n_kv, hq = n_q * hd; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * hn = ggml_mul(C, ggml_rms_norm(C, h, m.lm_rms_eps), w.attn_norm); - ggml_tensor * qp = ggml_mul_mat(C, w.Wq, hn); - ggml_tensor * kp = ggml_mul_mat(C, w.Wk, hn); - ggml_tensor * vp = ggml_mul_mat(C, w.Wv, hn); - ggml_tensor * qh = ggml_reshape_3d(C, qp, hd, n_q, seq); - ggml_tensor * kh = ggml_reshape_3d(C, kp, hd, n_kv, seq); - ggml_tensor * vh = ggml_reshape_3d(C, vp, hd, n_kv, seq); - ggml_tensor * qn = ggml_mul(C, ggml_rms_norm(C, qh, m.lm_rms_eps), w.q_norm); - ggml_tensor * kn = ggml_mul(C, ggml_rms_norm(C, kh, m.lm_rms_eps), w.k_norm); - ggml_tensor * qr = ggml_rope_ext(C, qn, positions, nullptr, (int) hd, GGML_ROPE_TYPE_NEOX, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * kr = ggml_rope_ext(C, kn, positions, nullptr, (int) hd, GGML_ROPE_TYPE_NEOX, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, mask, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hq, seq); - ggml_tensor * h_attn = ggml_add(C, h, ggml_mul_mat(C, w.Wo, att)); - ggml_tensor * hn2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); - ggml_tensor * gate = ggml_silu(C, ggml_mul_mat(C, w.Wgate, hn2)); - ggml_tensor * up = ggml_mul_mat(C, w.Wup, hn2); - return ggml_add(C, h_attn, ggml_mul_mat(C, w.Wdown, ggml_mul(C, gate, up))); -} - -ggml_tensor * build_vlsa_block(ggml_context * C, const VlsaLayerW & w, ggml_tensor * x, - int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) head_dim); - ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.n1w), w.n1b); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, n1), w.bv); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, head_dim, heads, seq), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq); - ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.n3w), w.n3b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} - -void dit_kv(ggml_context * C, const Gr00tN1d5ModelArch & m, const DitLayerW & w, ggml_tensor * kv, - ggml_tensor ** K_out, ggml_tensor ** V_out) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, Tkv = kv->ne[1]; - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, kv), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, kv), w.bv); - *K_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, Tkv), 0, 2, 1, 3)); - *V_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, Tkv), 1, 2, 0, 3)); -} - -ggml_tensor * build_dit_block(ggml_context * C, const Gr00tN1d5ModelArch & m, const DitLayerW & w, - ggml_tensor * h, ggml_tensor * temb, ggml_tensor * enc , - ggml_tensor * K_pre = nullptr, ggml_tensor * V_pre = nullptr) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, dim = m.dit_hidden, Tk = h->ne[1]; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * n = adaln(C, h, temb, w.adaln_w, w.adaln_b, dim, m.ln_eps); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n), w.bq); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, Tk), 0, 2, 1, 3)); - ggml_tensor * K, * V; - if (K_pre) { K = K_pre; V = V_pre; } - else { dit_kv(C, m, w, enc ? enc : n, &K, &V); } - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), dim, Tk); - ggml_tensor * h1 = ggml_add(C, h, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_norm(C, h1, m.ln_eps); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} - - bool load_config(const gguf_reader & g, Gr00tN1d5ModelArch & m, Config & cfg) { - auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; - auto F = [&](const char * k, float & dst) { if (g.has(k)) dst = g.f32(k); }; + auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; + auto F = [&](const char * k, float & dst) { if (g.has(k)) dst = g.f32(k); }; auto fk = [&](const char * s) { static char b[64]; std::snprintf(b, sizeof(b), "gr00t_n1_5.%s", s); return b; }; - U(fk("vit_hidden"), m.vit_hidden); U(fk("vit_layers"), m.vit_layers); U(fk("vit_heads"), m.vit_heads); U(fk("vit_inter"), m.vit_inter); - U(fk("image_size"), m.image_size); U(fk("patch_size"), m.patch_size); U(fk("n_img_tokens"), m.n_img_tokens); - U(fk("lm_hidden"), m.lm_hidden); U(fk("lm_layers_used"), m.lm_layers); U(fk("lm_q_heads"), m.n_q); U(fk("lm_kv_heads"), m.n_kv); - U(fk("lm_head_dim"), m.lm_head_dim); U(fk("lm_inter"), m.lm_inter); U(fk("vocab_size"), m.vocab); U(fk("image_token_index"), m.image_token_index); - U(fk("backbone_embedding_dim"), m.bb_embed_dim); U(fk("input_embedding_dim"), m.in_embed_dim); - U(fk("dit_hidden"), m.dit_hidden); U(fk("dit_heads"), m.dit_heads); U(fk("dit_head_dim"), m.dit_head_dim); U(fk("dit_layers"), m.dit_layers); U(fk("dit_interleave"), m.dit_interleave); - U(fk("vlsa_layers"), m.vlsa_layers); U(fk("vlsa_heads"), m.vlsa_heads); U(fk("vlsa_head_dim"), m.vlsa_head_dim); U(fk("vlsa_inter"), m.vlsa_inter); - U(fk("num_target_vision_tokens"), m.num_future); U(fk("action_horizon"), m.action_horizon); U(fk("action_dim"), m.action_dim); U(fk("max_state_dim"), m.max_state_dim); - U(fk("num_inference_timesteps"), m.num_steps); U(fk("num_timestep_buckets"), m.num_buckets); U(fk("max_num_embodiments"), m.max_embodiments); U(fk("max_seq_len"), m.max_seq_len); - F(fk("vit_ln_eps"), m.vit_ln_eps); F(fk("lm_rms_eps"), m.lm_rms_eps); F(fk("ln_eps"), m.ln_eps); F(fk("norm_out_eps"), m.norm_out_eps); F(fk("vlln_eps"), m.vlln_eps); - if (g.has(fk("lm_rope_theta"))) m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); - - m.embodiment_id = 24; + + U(fk("vit_hidden" ), m.vit.enc.cfg.hidden); + U(fk("vit_layers" ), m.vit_layers); + U(fk("vit_heads" ), m.vit.enc.cfg.heads); + U(fk("vit_inter" ), m.vit_inter); + U(fk("image_size" ), m.image_size); + U(fk("patch_size" ), m.patch_size); + U(fk("n_img_tokens" ), m.n_img_tokens); + U(fk("lm_hidden" ), m.lm.cfg.hidden); + U(fk("lm_layers_used" ), m.lm.cfg.layers); + U(fk("lm_q_heads" ), m.lm.cfg.n_q); + U(fk("lm_kv_heads" ), m.lm.cfg.n_kv); + U(fk("lm_head_dim" ), m.lm.cfg.head_dim); + U(fk("lm_inter" ), m.lm_inter); + U(fk("vocab_size" ), m.vocab); + U(fk("image_token_index"), m.image_token_index); + U(fk("backbone_embedding_dim"), m.bb_embed_dim); + U(fk("input_embedding_dim" ), m.in_embed_dim); + U(fk("dit_hidden" ), m.dit.cfg.hidden); + U(fk("dit_heads" ), m.dit.cfg.heads); + U(fk("dit_head_dim" ), m.dit.cfg.head_dim); + U(fk("dit_layers" ), m.dit.cfg.layers); + U(fk("dit_interleave" ), m.dit_interleave); + U(fk("vlsa_layers" ), m.vlsa_layers); + U(fk("vlsa_heads" ), m.vlsa.cfg.heads); + U(fk("vlsa_head_dim" ), m.vlsa.cfg.head_dim); + U(fk("num_target_vision_tokens"), m.num_future); + U(fk("action_horizon" ), m.action_horizon); + U(fk("action_dim" ), m.action_dim); + U(fk("max_state_dim" ), m.max_state_dim); + U(fk("num_inference_timesteps"), m.num_steps); + U(fk("num_timestep_buckets" ), m.num_buckets); + U(fk("max_num_embodiments" ), m.max_embodiments); + U(fk("max_seq_len" ), m.max_seq_len); + + F(fk("vit_ln_eps" ), m.vit.enc.cfg.ln_eps); + F(fk("lm_rms_eps" ), m.lm.cfg.rms_eps); + F(fk("ln_eps" ), m.dit.cfg.ln_eps); + F(fk("norm_out_eps" ), m.dit.cfg.norm_out_eps); + F(fk("vlln_eps" ), m.vlln_eps); + + if (g.has(fk("lm_rope_theta"))) m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); + + m.vit.enc.cfg.head_dim = m.vit.enc.cfg.hidden/m.vit.enc.cfg.heads; + m.vlsa.cfg.hidden = m.bb_embed_dim; + m.vlsa.cfg.ln_eps = m.dit.cfg.ln_eps; + m.lm.cfg.rope.n_dims = (int) m.lm.cfg.head_dim; + + m.aex.embodiment_id = 24; if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { - char * end = nullptr; long v = std::strtol(e, &end, 10); - if (end && *end == '\0') { m.embodiment_id = (int64_t) v; } - else { - const std::string js = g.str(fk("embodiment_tag_mapping")); - const std::string key = std::string("\"") + e + "\":"; + char * end = nullptr; + const long v = std::strtol(e, &end, 10); + if (end && *end == '\0') { + m.aex.embodiment_id = (int64_t) v; + } else { + const std::string js = g.str(fk("embodiment_tag_mapping")); + const std::string key = std::string("\"")+e+"\":"; const size_t p = js.find(key); - if (p != std::string::npos) m.embodiment_id = std::strtol(js.c_str() + p + key.size(), nullptr, 10); - else std::fprintf(stderr, "vla(gr00tn1d5): embodiment tag '%s' not in embodiment_tag_mapping; using id %lld\n", e, (long long) m.embodiment_id); + if (p != std::string::npos) m.aex.embodiment_id = std::strtol(js.c_str()+p+key.size(), nullptr, 10); + else std::fprintf(stderr, "vla(gr00tn1d5): embodiment tag '%s' not in embodiment_tag_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); } } - if (m.embodiment_id < 0 || m.embodiment_id >= m.max_embodiments) { std::fprintf(stderr, "vla(gr00tn1d5): embodiment id %lld out of range [0,%lld)\n", (long long) m.embodiment_id, (long long) m.max_embodiments); return false; } + if (m.aex.embodiment_id < 0 || m.aex.embodiment_id >= m.max_embodiments) { + std::fprintf(stderr, "vla(gr00tn1d5): embodiment id %lld out of range [0,%lld)\n", + (long long) m.aex.embodiment_id, (long long) m.max_embodiments); + return false; + } cfg = Config{}; - cfg.n_img = m.n_img_tokens; cfg.n_lang = m.max_seq_len; cfg.n_state = 1; - cfg.n_suffix = m.action_horizon; cfg.max_state_dim = m.max_state_dim; cfg.max_action_dim = m.action_dim; - cfg.real_state_dim = m.max_state_dim; cfg.real_action_dim = m.action_dim; - cfg.hidden = m.lm_hidden; cfg.n_q_heads = m.n_q; cfg.n_kv_heads = m.n_kv; cfg.head_dim = m.lm_head_dim; cfg.n_layers = m.lm_layers; - cfg.num_steps = (int) m.num_steps; cfg.rms_eps = m.lm_rms_eps; - cfg.rope_n_dims = (int) m.lm_head_dim; cfg.rope_mode = GGML_ROPE_TYPE_NEOX; cfg.rope_freq_base = m.lm_rope_base; + cfg.n_img = m.n_img_tokens; + cfg.n_lang = m.max_seq_len; + cfg.n_state = 1; + cfg.n_suffix = m.action_horizon; + cfg.max_state_dim = m.max_state_dim; + cfg.max_action_dim = m.action_dim; + cfg.real_state_dim = m.max_state_dim; + cfg.real_action_dim = m.action_dim; + cfg.hidden = m.lm.cfg.hidden; + cfg.n_q_heads = m.lm.cfg.n_q; + cfg.n_kv_heads = m.lm.cfg.n_kv; + cfg.head_dim = m.lm.cfg.head_dim; + cfg.n_layers = m.lm.cfg.layers; + cfg.num_steps = (int) m.num_steps; + cfg.rms_eps = m.lm.cfg.rms_eps; + cfg.rope_n_dims = (int) m.lm.cfg.head_dim; + cfg.rope_mode = GGML_ROPE_TYPE_NEOX; + cfg.rope_freq_base = m.lm.cfg.rope.freq_base; // Raw output: this arch expects the client to apply the dataset statistics // (see the --stats-json flag in eval/client). - cfg.denormalized = false; - cfg.norm_eps = 1e-8f; + cfg.denormalized = false; + cfg.norm_eps = 1e-8f; return true; } @@ -259,115 +204,55 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, std::printf("vla(gr00tn1d5): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; - if (!g.has("gr00t_n1_5.architecture")) { std::fprintf(stderr, "vla(gr00tn1d5): %s is not a gr00t_n1_5 GGUF\n", ckpt_path.c_str()); return nullptr; } + if (!g.has("gr00t_n1_5.architecture")) { + std::fprintf(stderr, "vla(gr00tn1d5): %s is not a gr00t_n1_5 GGUF\n", ckpt_path.c_str()); + return nullptr; + } if (!load_config(g, *m, m->cfg)) return nullptr; + std::printf("vla(gr00tn1d5): vit=%lldd×%lldL×%lldh n_img_tok=%lld lm=Qwen3 %lldd×%lldL (%lldq/%lldkv×%lld) " "dit=%lldL×%lldh×%lld(inner %lld) interleave=%lld vlsa=%lldL×%lldh×%lld in_emb=%lld horizon=%lld action_dim=%lld N_steps=%lld embodiment=%lld resident=%s\n", - (long long) m->vit_hidden, (long long) m->vit_layers, (long long) m->vit_heads, (long long) m->n_img_tokens, - (long long) m->lm_hidden, (long long) m->lm_layers, (long long) m->n_q, (long long) m->n_kv, (long long) m->lm_head_dim, - (long long) m->dit_layers, (long long) m->dit_heads, (long long) m->dit_head_dim, (long long) m->dit_hidden, (long long) m->dit_interleave, - (long long) m->vlsa_layers, (long long) m->vlsa_heads, (long long) m->vlsa_head_dim, (long long) m->in_embed_dim, - (long long) m->action_horizon, (long long) m->action_dim, (long long) m->num_steps, (long long) m->embodiment_id, + (long long) m->vit.enc.cfg.hidden, (long long) m->vit_layers, (long long) m->vit.enc.cfg.heads, (long long) m->n_img_tokens, + (long long) m->lm.cfg.hidden, (long long) m->lm.cfg.layers, (long long) m->lm.cfg.n_q, (long long) m->lm.cfg.n_kv, (long long) m->lm.cfg.head_dim, + (long long) m->dit.cfg.layers, (long long) m->dit.cfg.heads, (long long) m->dit.cfg.head_dim, (long long) m->dit.cfg.hidden, (long long) m->dit_interleave, + (long long) m->vlsa_layers, (long long) m->vlsa.cfg.heads, (long long) m->vlsa.cfg.head_dim, (long long) m->in_embed_dim, + (long long) m->action_horizon, (long long) m->action_dim, (long long) m->num_steps, (long long) m->aex.embodiment_id, m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); - { - const Backend b = backend_init("vla(gr00tn1d5)", m->n_threads); - if (!b.handle) { return nullptr; } - m->backend = b.handle; - } + const Backend b = backend_init("vla(gr00tn1d5)", m->n_threads); + if (!b.handle) return nullptr; + m->backend = b.handle; - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d5): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(gr00tn1d5): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; - - bool ok = true; - - m->vit_patch_w = mk("vit.patch_embd.weight", GGML_TYPE_F32); - m->vit_patch_b = mk_f32("vit.patch_embd.bias"); - m->vit_pos = mk_f32("vit.pos_embd"); - m->vit_post_ln_w = mk_f32("vit.post_ln.weight"); m->vit_post_ln_b = mk_f32("vit.post_ln.bias"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - ok &= w.ln1w&&w.ln1b&&w.ln2w&&w.ln2b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wfc1&&w.bfc1&&w.Wfc2&&w.bfc2; - } - m->mm_W = mk_mm("mm.fc.weight"); m->mm_b = mk_f32("mm.fc.bias"); - - m->lm_output_norm = mk_f32("vlm.output_norm.weight"); - m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vlm.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->lm[i]; - w.attn_norm=mk_f32(N("attn_norm.weight")); - w.Wq=mk_mm(N("attn_q.weight")); w.Wk=mk_mm(N("attn_k.weight")); w.Wv=mk_mm(N("attn_v.weight")); w.Wo=mk_mm(N("attn_o.weight")); - w.q_norm=mk_f32(N("attn_q_norm.weight")); w.k_norm=mk_f32(N("attn_k_norm.weight")); w.ffn_norm=mk_f32(N("ffn_norm.weight")); - w.Wgate=mk_mm(N("ffn_gate.weight")); w.Wup=mk_mm(N("ffn_up.weight")); w.Wdown=mk_mm(N("ffn_down.weight")); - ok &= w.attn_norm&&w.Wq&&w.Wk&&w.Wv&&w.Wo&&w.q_norm&&w.k_norm&&w.ffn_norm&&w.Wgate&&w.Wup&&w.Wdown; - } - m->vlln_w=mk_f32("aex.vlln.weight"); m->vlln_b=mk_f32("aex.vlln.bias"); - m->vlsa.resize(m->vlsa_layers); - for (int64_t i = 0; i < m->vlsa_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "aex.vlsa.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vlsa[i]; - w.n1w=mk_f32(N("norm1.weight")); w.n1b=mk_f32(N("norm1.bias")); w.n3w=mk_f32(N("norm3.weight")); w.n3b=mk_f32(N("norm3.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.n1w&&w.n1b&&w.n3w&&w.n3b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } + WeightLoader L("gr00tn1d5", g, m->ctx_weights, m->matmul_type); + + m->vit.declare(L, "vit", m->vit_layers); + m->mm_W = L.gemm("mm.fc.weight"); + m->mm_b = L.f32 ("mm.fc.bias"); + + m->lm.declare(L, "vlm"); + + m->vlln_w = L.f32("aex.vlln.weight"); + m->vlln_b = L.f32("aex.vlln.bias"); + m->vlsa.declare(L, "aex.vlsa", m->vlsa_layers, EncNames{"norm1", "norm3", "ff0", "ff2"}); + + m->aex.declare(L, "aex"); + m->future_tokens = L.f32("aex.future_tokens"); + m->dit.declare(L, "aex.dit"); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; - m->se_l1W=mk_f32("aex.state_enc.l1.W"); m->se_l1b=mk_f32("aex.state_enc.l1.b"); m->se_l2W=mk_f32("aex.state_enc.l2.W"); m->se_l2b=mk_f32("aex.state_enc.l2.b"); - m->ae_W1W=mk_f32("aex.act_enc.W1.W"); m->ae_W1b=mk_f32("aex.act_enc.W1.b"); m->ae_W2W=mk_f32("aex.act_enc.W2.W"); m->ae_W2b=mk_f32("aex.act_enc.W2.b"); m->ae_W3W=mk_f32("aex.act_enc.W3.W"); m->ae_W3b=mk_f32("aex.act_enc.W3.b"); - m->ad_l1W=mk_f32("aex.act_dec.l1.W"); m->ad_l1b=mk_f32("aex.act_dec.l1.b"); m->ad_l2W=mk_f32("aex.act_dec.l2.W"); m->ad_l2b=mk_f32("aex.act_dec.l2.b"); - m->future_tokens=mk_f32("aex.future_tokens"); m->pos_embd=mk_f32("aex.pos_embd"); - m->te_l1W=mk_mm("aex.dit.time_emb.l1.weight"); m->te_l1b=mk_f32("aex.dit.time_emb.l1.bias"); m->te_l2W=mk_mm("aex.dit.time_emb.l2.weight"); m->te_l2b=mk_f32("aex.dit.time_emb.l2.bias"); - m->dit.resize(m->dit_layers); - for (int64_t i = 0; i < m->dit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "aex.dit.%lld.%s", (long long) i, s); return p; }; - auto & w = m->dit[i]; - w.adaln_w=mk_mm(N("adaln.weight")); w.adaln_b=mk_f32(N("adaln.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.adaln_w&&w.adaln_b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } - m->po1W=mk_mm("aex.dit.proj_out1.weight"); m->po1b=mk_f32("aex.dit.proj_out1.bias"); m->po2W=mk_mm("aex.dit.proj_out2.weight"); m->po2b=mk_f32("aex.dit.proj_out2.bias"); - ok &= m->vit_patch_w&&m->vit_patch_b&&m->vit_pos&&m->vit_post_ln_w&&m->vit_post_ln_b&&m->mm_W&&m->mm_b&&m->lm_output_norm&& - m->vlln_w&&m->vlln_b&&m->se_l1W&&m->se_l1b&&m->se_l2W&&m->se_l2b&&m->ae_W1W&&m->ae_W1b&&m->ae_W2W&&m->ae_W2b&&m->ae_W3W&&m->ae_W3b&& - m->ad_l1W&&m->ad_l1b&&m->ad_l2W&&m->ad_l2b&&m->future_tokens&&m->pos_embd&&m->te_l1W&&m->te_l1b&&m->te_l2W&&m->te_l2b&&m->po1W&&m->po1b&&m->po2W&&m->po2b; - if (!ok) { std::fprintf(stderr, "vla(gr00tn1d5): weight tensor setup failed\n"); return nullptr; } - - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(gr00tn1d5): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(gr00tn1d5): failed to load %s (%zu vs %zu bytes)\n", ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } std::printf("vla(gr00tn1d5): weights resident in %.2f GiB (%s) - incl. SigLIP vision tower; embodiment id %lld\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", (long long) m->embodiment_id); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), + m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", (long long) m->aex.embodiment_id); return m; } @@ -375,178 +260,163 @@ std::vector Gr00tN1d5ModelArch::predict(const Inputs& in) { const auto t0 = std::chrono::steady_clock::now(); stats = Stats{}; - const int64_t H = lm_hidden, K = n_img_tokens, E = in_embed_dim; - const int64_t Nsa = 1 + num_future + action_horizon; + const int64_t H = lm.cfg.hidden; + const int64_t K = n_img_tokens; + const int64_t E = in_embed_dim; + const int64_t AD = action_dim; + const int64_t AH = action_horizon; + const int64_t Nsa = 1+num_future+AH; int64_t n_views = 0; std::vector img_emb_host; const float * img_emb_ptr = nullptr; + if (in.precomputed_img_emb && in.n_img_views > 0) { - n_views = in.n_img_views; + n_views = in.n_img_views; img_emb_ptr = in.precomputed_img_emb; } else if (in.images && in.n_images > 0) { n_views = in.n_images; - img_emb_host.assign((size_t) n_views * K * H, 0.0f); - ggml_context * VC = vision_scratch.reset((size_t) 64 * 1024 * 1024); + img_emb_host.assign((size_t) n_views*K*H, 0.0f); + + ggml_context * VC = vision_scratch.reset((size_t) 64*1024*1024); if (!VC) { std::fprintf(stderr, "vla(gr00tn1d5): ggml_init(vision ctx) failed\n"); return {}; } - const int64_t grid = image_size / patch_size; - ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, image_size, image_size, 3); ggml_set_input(t_px); - ggml_tensor * conv = ggml_conv_2d(VC, vit_patch_w, t_px, (int) patch_size, (int) patch_size, 0, 0, 1, 1); - ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); - ggml_tensor * h = ggml_add(VC, ggml_add(VC, patches, vit_patch_b), vit_pos); - for (int64_t i = 0; i < vit_layers; ++i) h = build_siglip_layer(VC, vit[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps); - h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit_post_ln_w), vit_post_ln_b); - ggml_tensor * vit_emb = ggml_add(VC, ggml_mul_mat(VC, mm_W, h), mm_b); + + const int64_t grid = image_size/patch_size; + ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, image_size, image_size, 3); + ggml_set_input(t_px); + + ggml_tensor * h = vit.build(VC, vit.embed_conv(VC, t_px, patch_size, grid), K); + ggml_tensor * vit_emb = linear(VC, mm_W, mm_b, h); ggml_set_output(vit_emb); + ggml_cgraph * vg = ggml_new_graph_custom(VC, 8192, false); ggml_build_forward_expand(vg, vit_emb); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(gr00tn1d5): vision gallocr alloc failed\n"); return {}; } + const auto tv0 = std::chrono::steady_clock::now(); std::vector chw; for (int64_t v = 0; v < n_views; ++v) { - if (!preprocess_image_chw("gr00tn1d5", in.images[v], image_size, chw)) { return {}; } + if (!preprocess_image_chw("gr00tn1d5", in.images[v], image_size, chw)) return {}; ggml_backend_tensor_set(t_px, chw.data(), 0, ggml_nbytes(t_px)); - if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d5): vision compute failed\n"); return {}; } - ggml_backend_tensor_get(vit_emb, img_emb_host.data() + v * K * H, 0, ggml_nbytes(vit_emb)); + if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d5): vision compute failed\n"); + return {}; + } + ggml_backend_tensor_get(vit_emb, img_emb_host.data()+v*K*H, 0, ggml_nbytes(vit_emb)); } - stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); img_emb_ptr = img_emb_host.data(); } else { - std::fprintf(stderr, "vla(gr00tn1d5): no images and no precomputed_img_emb in the request\n"); return {}; - } - const int64_t n_img = n_views * K; - - std::vector input_ids; - int64_t n_lang_img = 0; - for (int j = 0; j < in.n_lang; ++j) if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_lang_img; - if (n_lang_img > 0) { - if (n_lang_img != n_img) { - std::fprintf(stderr, "vla(gr00tn1d5): lang_tokens has %lld placeholders but %lld ViT embeds " - "(n_views=%lld × K=%lld)\n", (long long) n_lang_img, (long long) n_img, (long long) n_views, (long long) K); - return {}; - } - input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); - } else { - input_ids.reserve(n_img + in.n_lang); - for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); - } - const int64_t SEQ = (int64_t) input_ids.size(); - if (SEQ > max_seq_len) { std::fprintf(stderr, "vla(gr00tn1d5): prompt too long (%lld > %lld)\n", (long long) SEQ, (long long) max_seq_len); return {}; } - - std::vector inputs_embeds((size_t) SEQ * H); - if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), H)) return {}; - { int64_t k = 0; - for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { - if (k >= n_img) { std::fprintf(stderr, "vla(gr00tn1d5): more tokens than ViT embeds\n"); return {}; } - std::memcpy(inputs_embeds.data() + p * H, img_emb_ptr + k * H, H * sizeof(float)); ++k; - } + std::fprintf(stderr, "vla(gr00tn1d5): no images and no precomputed_img_emb in the request\n"); + return {}; } + const int64_t n_img = n_views*K; + + Prompt prompt; + if (!build_prompt("gr00tn1d5", in, n_img, (int32_t) image_token_index, max_seq_len, prompt)) return {}; + const int64_t SEQ = prompt.len(); - const int64_t AD = action_dim, AH = action_horizon; - std::vector x_init((size_t) AH * AD); - if (in.noise) std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); - else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); for (auto & v : x_init) v = nd(rng); } + std::vector inputs_embeds; + if (!fetch_embeds("gr00tn1d5", io, prompt, img_emb_ptr, H, inputs_embeds)) return {}; + + std::vector x_init; + init_noise(in, (size_t) AH*AD, x_init); // LM + VLSA + DiT graph depends only on the padded length and step count. const MainKey mkey{ SEQ, num_steps }; - const bool built = main_graph.ensure(backend, mkey, (size_t) 128 * 1024 * 1024, + const bool built = main_graph.ensure(backend, mkey, (size_t) 128*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { - ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); - ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ); ggml_set_input(t_pos); - ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); - ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1);ggml_set_input(t_state); - ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); - std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { - t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); - t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, 256); ggml_set_input(t_tproj[s]); - } - - ggml_tensor * h = t_embeds; - for (int64_t i = 0; i < lm_layers; ++i) h = build_qwen3_layer(C, *this, lm[i], h, t_pos, t_lmmask, SEQ); - ggml_tensor * eagle = ggml_mul(C, ggml_rms_norm(C, h, lm_rms_eps), lm_output_norm); + ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); + ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ); ggml_set_input(t_pos); + ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); + ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1); ggml_set_input(t_state); + ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); + + std::vector t_tau(num_steps), t_tproj(num_steps); + for (int64_t s = 0; s < num_steps; ++s) { + t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); + t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, 256); ggml_set_input(t_tproj[s]); + } - ggml_tensor * vl = ggml_add(C, ggml_mul(C, ggml_norm(C, eagle, vlln_eps), vlln_w), vlln_b); - for (int64_t i = 0; i < vlsa_layers; ++i) vl = build_vlsa_block(C, vlsa[i], vl, SEQ, vlsa_heads, vlsa_head_dim, bb_embed_dim, ln_eps); - ggml_tensor * vl_embs = vl; + ggml_tensor * eagle = lm.build(C, t_embeds, t_pos, t_lmmask, SEQ); + ggml_tensor * vl = layer_norm(C, eagle, vlln_w, vlln_b, vlln_eps); + ggml_tensor * vl_embs = vlsa.build(C, vl, SEQ); - ggml_tensor * state_features = cat_linear(C, se_l2W, se_l2b, embodiment_id, ggml_relu(C, cat_linear(C, se_l1W, se_l1b, embodiment_id, t_state))); + ggml_tensor * state_features = aex.encode_state(C, t_state); - std::vector Kc(dit_layers, nullptr), Vc(dit_layers, nullptr); - for (int64_t i = 0; i < dit_layers; ++i) { - if (dit_interleave && (i % 2 == 1)) continue; - dit_kv(C, *this, dit[i], vl_embs, &Kc[i], &Vc[i]); - } + std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); + for (int64_t i = 0; i < dit.cfg.layers; ++i) { + if (dit_interleave && (i%2 == 1)) continue; + dit.kv(C, dit.blk[i], vl_embs, &Kc[i], &Vc[i]); + } - const float dt = 1.0f / (float) num_steps; - ggml_tensor * actions = t_x0; - for (int64_t s = 0; s < num_steps; ++s) { + const float dt = 1.0f/(float) num_steps; + ggml_tensor * actions = t_x0; + for (int64_t s = 0; s < num_steps; ++s) { + ggml_tensor * temb = dit.time_emb(C, t_tproj[s]); + ggml_tensor * af = aex.encode_action(C, actions, t_tau[s], E, AH); + ggml_tensor * hh = ggml_concat(C, ggml_concat(C, state_features, future_tokens, 1), af, 1); + + for (int64_t i = 0; i < dit.cfg.layers; ++i) { + ggml_tensor * enc = (dit_interleave && (i%2 == 1)) ? nullptr : vl_embs; + hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); + } + + ggml_tensor * pred = aex.decode(C, dit.proj_out(C, hh, temb)); + ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t)(Nsa-AH)*pred->nb[1])); + actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); + } + ggml_set_name(actions, "action_pred"); + ggml_set_output(actions); - ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, te_l1W, t_tproj[s]), te_l1b))), te_l2b); + gio.t_embeds=t_embeds; gio.t_pos=t_pos; gio.t_lmmask=t_lmmask; gio.t_state=t_state; + gio.t_x0=t_x0; gio.t_tau=t_tau; gio.t_tproj=t_tproj; gio.actions=actions; - ggml_tensor * a_emb = cat_linear(C, ae_W1W, ae_W1b, embodiment_id, actions); - ggml_tensor * x_w2 = ggml_silu(C, cat_linear(C, ae_W2W, ae_W2b, embodiment_id, ggml_concat(C, a_emb, t_tau[s], 0))); - ggml_tensor * af = ggml_add(C, cat_linear(C, ae_W3W, ae_W3b, embodiment_id, x_w2), ggml_view_2d(C, pos_embd, E, AH, pos_embd->nb[1], 0)); + ggml_cgraph * gf = ggml_new_graph_custom(C, 32768, false); + ggml_build_forward_expand(gf, actions); + return gf; + }); + if (!built) { std::fprintf(stderr, "vla(gr00tn1d5): main graph build failed\n"); return {}; } - ggml_tensor * sa = ggml_concat(C, ggml_concat(C, state_features, future_tokens, 1), af, 1); + MainIO & gio = main_graph.io(); - ggml_tensor * hh = sa; - for (int64_t i = 0; i < dit_layers; ++i) { - ggml_tensor * enc = (dit_interleave && (i % 2 == 1)) ? nullptr : vl_embs; - hh = build_dit_block(C, *this, dit[i], hh, temb, enc, Kc[i], Vc[i]); - } + ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); - ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, po1W, ggml_silu(C, temb)), po1b); - ggml_tensor * sh = ggml_view_1d(C, po, dit_hidden, 0), * sc = ggml_view_1d(C, po, dit_hidden, (size_t) dit_hidden * sizeof(float)); - ggml_tensor * hn = ggml_norm(C, hh, norm_out_eps); - ggml_tensor * h_mod = ggml_add(C, ggml_add(C, hn, ggml_mul(C, hn, sc)), sh); - ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, po2W, h_mod), po2b); + std::vector pp(SEQ); + for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; + ggml_backend_tensor_set(gio.t_pos, pp.data(), 0, ggml_nbytes(gio.t_pos)); - ggml_tensor * pred = cat_linear(C, ad_l2W, ad_l2b, embodiment_id, ggml_relu(C, cat_linear(C, ad_l1W, ad_l1b, embodiment_id, model_output))); - ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t) (Nsa - AH) * pred->nb[1])); - actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); - } - ggml_set_name(actions, "action_pred"); ggml_set_output(actions); + std::vector mask; + build_causal_mask(SEQ, mask); + ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); - gio.t_embeds=t_embeds; gio.t_pos=t_pos; gio.t_lmmask=t_lmmask; gio.t_state=t_state; - gio.t_x0=t_x0; gio.t_tau=t_tau; gio.t_tproj=t_tproj; gio.actions=actions; + std::vector st(max_state_dim, 0.0f); + for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; + ggml_backend_tensor_set(gio.t_state, st.data(), 0, ggml_nbytes(gio.t_state)); - ggml_cgraph * gf = ggml_new_graph_custom(C, 32768, false); - ggml_build_forward_expand(gf, actions); - return gf; - }); - if (!built) { std::fprintf(stderr, "vla(gr00tn1d5): main graph build failed\n"); return {}; } + ggml_backend_tensor_set(gio.t_x0, x_init.data(), 0, ggml_nbytes(gio.t_x0)); - MainIO & gio = main_graph.io(); - ggml_cgraph * gf = main_graph.graph(); - ggml_tensor * t_embeds = gio.t_embeds, * t_pos = gio.t_pos, * t_lmmask = gio.t_lmmask; - ggml_tensor * t_state = gio.t_state, * t_x0 = gio.t_x0, * actions = gio.actions; - std::vector & t_tau = gio.t_tau; std::vector & t_tproj = gio.t_tproj; - - ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); - { std::vector pp(SEQ); for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } - { std::vector mk((size_t) SEQ * SEQ); const float NEG = -std::numeric_limits::infinity(); - for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ + kv] = (kv <= q) ? 0.0f : NEG; - ggml_backend_tensor_set(t_lmmask, mk.data(), 0, ggml_nbytes(t_lmmask)); } - { std::vector st(max_state_dim, 0.0f); for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); } - ggml_backend_tensor_set(t_x0, x_init.data(), 0, ggml_nbytes(t_x0)); for (int64_t s = 0; s < num_steps; ++s) { - const int64_t bucket = (int64_t) ((double) s / (double) num_steps * (double) num_buckets); - std::vector tau, tpr; action_sinusoid(bucket, E, AH, tau); timesteps_proj(bucket, tpr); - ggml_backend_tensor_set(t_tau[s], tau.data(), 0, ggml_nbytes(t_tau[s])); - ggml_backend_tensor_set(t_tproj[s], tpr.data(), 0, ggml_nbytes(t_tproj[s])); + const int64_t bucket = (int64_t) ((double) s/(double) num_steps*(double) num_buckets); + std::vector tau, tpr; + action_sinusoid(bucket, E, AH, tau); + timesteps_proj(bucket, tpr); + ggml_backend_tensor_set(gio.t_tau[s], tau.data(), 0, ggml_nbytes(gio.t_tau[s])); + ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } const auto tc0 = std::chrono::steady_clock::now(); - const ggml_status st = ggml_backend_graph_compute(backend, gf); + const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); - if (st != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d5): graph compute failed (%d)\n", (int) st); return {}; } - stats.ms_inference = std::chrono::duration(tc1 - tc0).count(); + if (status != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d5): graph compute failed (%d)\n", (int) status); + return {}; + } + stats.ms_inference = std::chrono::duration(tc1-tc0).count(); - std::vector out((size_t) AH * AD); - ggml_backend_tensor_get(actions, out.data(), 0, out.size() * sizeof(float)); - stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + std::vector out((size_t) AH*AD); + ggml_backend_tensor_get(gio.actions, out.data(), 0, out.size()*sizeof(float)); + stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now()-t0).count(); return out; } diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index fdafdaa..980cbe5 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -12,51 +12,53 @@ // See the License for the specific language governing permissions and // limitations under the License. +// NVIDIA Isaac GR00T N1.6: SigLIP2 tower with a GEMM patch embed and a +// pixel-shuffled MLP connector, a Qwen3 backbone, and an AlternateVL DiT head +// that cross-attends text and image tokens on alternating blocks. + #include "arch.h" +#include "backend.h" +#include "env_flag.h" +#include "gguf_reader.h" +#include "layers/embed.h" +#include "layers/ffn.h" +#include "layers/linear.h" +#include "layers/norm.h" #include "model.h" +#include "modules/action_expert.h" +#include "modules/dit_head.h" +#include "modules/preprocess.h" +#include "modules/prompt.h" +#include "modules/qwen3_lm.h" +#include "modules/siglip_vit.h" +#include "scratch_ctx.h" #include "ggml.h" -#include "ggml-cpu.h" #include "ggml-backend.h" -#include "backend.h" -#include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" -#include "models/dit_common.h" -#include "env_flag.h" #include -#include #include #include #include #include -#include #include #include #include #include namespace vla { -namespace { - - -struct SigLipLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; -struct Qwen3LayerW { ggml_tensor *attn_norm,*Wq,*Wk,*Wv,*Wo,*q_norm,*k_norm,*ffn_norm,*Wgate,*Wup,*Wdown; }; -struct DitLayerW { ggml_tensor *adaln_w,*adaln_b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; }; - -} struct Gr00tN1d6ModelArch : public ModelArchBase { Gr00tN1d6ModelArch() : ModelArchBase(Arch::GR00T_N1_6) {} ~Gr00tN1d6ModelArch() override; - std::string gguf_path; // Opened once at load: reopening per predict re-parses the whole GGUF header. gguf_reader io{"gr00tn1d6"}; ggml_backend_t backend = nullptr; int n_threads = default_cpu_threads(); ggml_context * ctx_weights = nullptr; + ggml_backend_buffer_t weight_buf = nullptr; + ggml_type matmul_type = GGML_TYPE_F32; scratch_ctx vision_scratch; scratch_ctx merge_scratch; @@ -72,198 +74,123 @@ struct Gr00tN1d6ModelArch : public ModelArchBase { std::vector t_tau, t_tproj; }; graph_cache main_graph; - ggml_backend_buffer_t weight_buf = nullptr; - ggml_type matmul_type = GGML_TYPE_F32; - int64_t vit_hidden=1152, vit_layers=27, vit_heads=16, vit_inter=4304, image_size=224, patch_size=14; + SigLipTower vit; + Qwen3LM lm; + ActionExpert aex; + DitHead dit; + ggml_tensor *mm_ln_w=nullptr, *mm_ln_b=nullptr; + ggml_tensor *mm_fc1_w=nullptr, *mm_fc1_b=nullptr, *mm_fc2_w=nullptr, *mm_fc2_b=nullptr; + ggml_tensor *vlln_w=nullptr, *vlln_b=nullptr; + + int64_t vit_layers=27, vit_inter=4304, image_size=224, patch_size=14; int64_t vit_num_patches=256, n_img_tokens=64, vit_pixel_shuffle=2, mlp_inner=4608; - int64_t lm_hidden=2048, lm_layers=16, n_q=16, n_kv=8, lm_head_dim=128, lm_inter=6144, vocab=151680, image_token_index=151669; - int64_t bb_embed_dim=2048, in_embed_dim=1536, dit_hidden=1536, dit_heads=32, dit_head_dim=48, dit_layers=32, dit_interleave=1, attend_text_every_n=2; + int64_t lm_inter=6144, vocab=151680, image_token_index=151669; + int64_t bb_embed_dim=2048, in_embed_dim=1536, dit_interleave=1, attend_text_every_n=2; int64_t action_horizon=50, action_dim=128, max_state_dim=128; int64_t num_steps=4, num_buckets=1000, max_embodiments=32, max_seq_len=1024; - float vit_ln_eps=1e-6f, lm_rms_eps=1e-6f, ln_eps=1e-5f, norm_out_eps=1e-6f, vlln_eps=1e-5f, connector_ln_eps=1e-5f, lm_rope_base=1000000.0f; - int64_t embodiment_id = 20; - - ggml_tensor *vit_patch_w=nullptr,*vit_patch_b=nullptr,*vit_pos=nullptr,*vit_post_ln_w=nullptr,*vit_post_ln_b=nullptr; - std::vector vit; - ggml_tensor *mm_ln_w=nullptr,*mm_ln_b=nullptr,*mm_fc1_w=nullptr,*mm_fc1_b=nullptr,*mm_fc2_w=nullptr,*mm_fc2_b=nullptr; - - ggml_tensor *lm_output_norm=nullptr; - std::vector lm; - - ggml_tensor *vlln_w=nullptr,*vlln_b=nullptr; - ggml_tensor *se_l1W=nullptr,*se_l1b=nullptr,*se_l2W=nullptr,*se_l2b=nullptr; - ggml_tensor *ae_W1W=nullptr,*ae_W1b=nullptr,*ae_W2W=nullptr,*ae_W2b=nullptr,*ae_W3W=nullptr,*ae_W3b=nullptr; - ggml_tensor *ad_l1W=nullptr,*ad_l1b=nullptr,*ad_l2W=nullptr,*ad_l2b=nullptr; - ggml_tensor *pos_embd=nullptr; - ggml_tensor *te_l1W=nullptr,*te_l1b=nullptr,*te_l2W=nullptr,*te_l2b=nullptr; - std::vector dit; - ggml_tensor *po1W=nullptr,*po1b=nullptr,*po2W=nullptr,*po2b=nullptr; + float vlln_eps=1e-5f, connector_ln_eps=1e-5f; std::vector predict(const Inputs& in) override; }; namespace { -ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, - int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const int64_t nv = x->ne[2]; - const float scale = 1.0f / std::sqrt((float) head_dim); - ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, n1), w.bv); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, q, head_dim, heads, seq, nv), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, k, head_dim, heads, seq, nv), 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, v, head_dim, heads, seq, nv), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_3d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq, nv); - ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); - return ggml_add(C, h1, ff); -} - -ggml_tensor * build_qwen3_layer(ggml_context * C, const Gr00tN1d6ModelArch & m, const Qwen3LayerW & w, - ggml_tensor * h, ggml_tensor * positions, ggml_tensor * mask, int64_t seq) { - const int64_t hd = m.lm_head_dim, n_q = m.n_q, n_kv = m.n_kv, hq = n_q * hd; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * hn = ggml_mul(C, ggml_rms_norm(C, h, m.lm_rms_eps), w.attn_norm); - ggml_tensor * qp = ggml_mul_mat(C, w.Wq, hn); - ggml_tensor * kp = ggml_mul_mat(C, w.Wk, hn); - ggml_tensor * vp = ggml_mul_mat(C, w.Wv, hn); - ggml_tensor * qh = ggml_reshape_3d(C, qp, hd, n_q, seq); - ggml_tensor * kh = ggml_reshape_3d(C, kp, hd, n_kv, seq); - ggml_tensor * vh = ggml_reshape_3d(C, vp, hd, n_kv, seq); - ggml_tensor * qn = ggml_mul(C, ggml_rms_norm(C, qh, m.lm_rms_eps), w.q_norm); - ggml_tensor * kn = ggml_mul(C, ggml_rms_norm(C, kh, m.lm_rms_eps), w.k_norm); - ggml_tensor * qr = ggml_rope_ext(C, qn, positions, nullptr, (int) hd, GGML_ROPE_TYPE_NEOX, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * kr = ggml_rope_ext(C, kn, positions, nullptr, (int) hd, GGML_ROPE_TYPE_NEOX, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, mask, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hq, seq); - ggml_tensor * h_attn = ggml_add(C, h, ggml_mul_mat(C, w.Wo, att)); - ggml_tensor * hn2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); - ggml_tensor * gate = ggml_silu(C, ggml_mul_mat(C, w.Wgate, hn2)); - ggml_tensor * up = ggml_mul_mat(C, w.Wup, hn2); - return ggml_add(C, h_attn, ggml_mul_mat(C, w.Wdown, ggml_mul(C, gate, up))); -} - -void dit_kv(ggml_context * C, const Gr00tN1d6ModelArch & m, const DitLayerW & w, ggml_tensor * kv, - ggml_tensor ** K_out, ggml_tensor ** V_out) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, Tkv = kv->ne[1]; - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, kv), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, kv), w.bv); - *K_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, Tkv), 0, 2, 1, 3)); - *V_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, Tkv), 1, 2, 0, 3)); -} - -ggml_tensor * build_dit_block(ggml_context * C, const Gr00tN1d6ModelArch & m, const DitLayerW & w, - ggml_tensor * h, ggml_tensor * temb, ggml_tensor * enc , - ggml_tensor * K_pre = nullptr, ggml_tensor * V_pre = nullptr) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, dim = m.dit_hidden, Tk = h->ne[1]; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * n = adaln(C, h, temb, w.adaln_w, w.adaln_b, dim, m.ln_eps); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n), w.bq); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, Tk), 0, 2, 1, 3)); - ggml_tensor * K, * V; - if (K_pre) { K = K_pre; V = V_pre; } - else { dit_kv(C, m, w, enc ? enc : n, &K, &V); } - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), dim, Tk); - ggml_tensor * h1 = ggml_add(C, h, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_norm(C, h1, m.ln_eps); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} - -bool preprocess_image_patches(const ImageView & v, int64_t side, int64_t ps, std::vector & out) { - if (v.w != (int) side || v.h != (int) side || !v.data) { - std::fprintf(stderr, "vla(gr00tn1d6): image view is %dx%d, expected %lldx%lld\n", v.w, v.h, (long long) side, (long long) side); return false; - } - const int64_t grid = side / ps, pd = 3 * ps * ps, np = grid * grid; - out.assign((size_t) pd * np, 0.0f); - auto px = [&](int64_t r, int64_t c, int64_t ch) -> float { - if (v.format == PixelFormat::U8) return ((const uint8_t *) v.data)[(r * side + c) * 3 + ch] / 255.0f; - return ((const float *) v.data)[(r * side + c) * 3 + ch]; - }; - for (int64_t row = 0; row < grid; ++row) - for (int64_t col = 0; col < grid; ++col) { - const int64_t t = row * grid + col; - for (int64_t ph = 0; ph < ps; ++ph) - for (int64_t pw = 0; pw < ps; ++pw) - for (int64_t ch = 0; ch < 3; ++ch) - out[t * pd + ph * ps * 3 + pw * 3 + ch] = px(row * ps + ph, col * ps + pw, ch) * 2.0f - 1.0f; - } - return true; -} - -void pixel_shuffle_back(const float * src, int64_t grid, int64_t hidden, int64_t r, float * dst) { - const int64_t g2 = grid / r, c4 = hidden * r * r; - for (int64_t y = 0; y < g2; ++y) - for (int64_t x = 0; x < g2; ++x) { - const int64_t t = y * g2 + x; - for (int64_t c = 0; c < hidden; ++c) - for (int64_t i = 0; i < r; ++i) - for (int64_t j = 0; j < r; ++j) { - const int64_t pp = (r * y + i) * grid + (r * x + j); - const int64_t cp = c * r * r + i * r + j; - dst[t * c4 + cp] = src[pp * hidden + c]; - } - } -} - bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { - auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; - auto F = [&](const char * k, float & dst) { if (g.has(k)) dst = g.f32(k); }; + auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; + auto F = [&](const char * k, float & dst) { if (g.has(k)) dst = g.f32(k); }; auto fk = [&](const char * s) { static char b[64]; std::snprintf(b, sizeof(b), "gr00t_n1_6.%s", s); return b; }; - U(fk("vit_hidden"), m.vit_hidden); U(fk("vit_layers"), m.vit_layers); U(fk("vit_heads"), m.vit_heads); U(fk("vit_inter"), m.vit_inter); - U(fk("image_size"), m.image_size); U(fk("patch_size"), m.patch_size); - U(fk("vit_num_patches"), m.vit_num_patches); U(fk("n_img_tokens"), m.n_img_tokens); U(fk("vit_pixel_shuffle"), m.vit_pixel_shuffle); U(fk("mlp_connector_inner"), m.mlp_inner); - U(fk("lm_hidden"), m.lm_hidden); U(fk("lm_layers_used"), m.lm_layers); U(fk("lm_q_heads"), m.n_q); U(fk("lm_kv_heads"), m.n_kv); - U(fk("lm_head_dim"), m.lm_head_dim); U(fk("lm_inter"), m.lm_inter); U(fk("vocab_size"), m.vocab); U(fk("image_token_index"), m.image_token_index); - U(fk("backbone_embedding_dim"), m.bb_embed_dim); U(fk("input_embedding_dim"), m.in_embed_dim); - U(fk("dit_hidden"), m.dit_hidden); U(fk("dit_heads"), m.dit_heads); U(fk("dit_head_dim"), m.dit_head_dim); U(fk("dit_layers"), m.dit_layers); U(fk("dit_interleave"), m.dit_interleave); - U(fk("attend_text_every_n_blocks"), m.attend_text_every_n); - U(fk("action_horizon"), m.action_horizon); U(fk("action_dim"), m.action_dim); U(fk("max_state_dim"), m.max_state_dim); - U(fk("num_inference_timesteps"), m.num_steps); U(fk("num_timestep_buckets"), m.num_buckets); U(fk("max_num_embodiments"), m.max_embodiments); U(fk("max_seq_len"), m.max_seq_len); - F(fk("vit_ln_eps"), m.vit_ln_eps); F(fk("lm_rms_eps"), m.lm_rms_eps); F(fk("ln_eps"), m.ln_eps); F(fk("norm_out_eps"), m.norm_out_eps); F(fk("vlln_eps"), m.vlln_eps); F(fk("connector_ln_eps"), m.connector_ln_eps); - if (g.has(fk("lm_rope_theta"))) m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); - m.embodiment_id = 20; + U(fk("vit_hidden" ), m.vit.enc.cfg.hidden); + U(fk("vit_layers" ), m.vit_layers); + U(fk("vit_heads" ), m.vit.enc.cfg.heads); + U(fk("vit_inter" ), m.vit_inter); + U(fk("image_size" ), m.image_size); + U(fk("patch_size" ), m.patch_size); + U(fk("vit_num_patches" ), m.vit_num_patches); + U(fk("n_img_tokens" ), m.n_img_tokens); + U(fk("vit_pixel_shuffle"), m.vit_pixel_shuffle); + U(fk("mlp_connector_inner"), m.mlp_inner); + U(fk("lm_hidden" ), m.lm.cfg.hidden); + U(fk("lm_layers_used" ), m.lm.cfg.layers); + U(fk("lm_q_heads" ), m.lm.cfg.n_q); + U(fk("lm_kv_heads" ), m.lm.cfg.n_kv); + U(fk("lm_head_dim" ), m.lm.cfg.head_dim); + U(fk("lm_inter" ), m.lm_inter); + U(fk("vocab_size" ), m.vocab); + U(fk("image_token_index"), m.image_token_index); + U(fk("backbone_embedding_dim"), m.bb_embed_dim); + U(fk("input_embedding_dim" ), m.in_embed_dim); + U(fk("dit_hidden" ), m.dit.cfg.hidden); + U(fk("dit_heads" ), m.dit.cfg.heads); + U(fk("dit_head_dim" ), m.dit.cfg.head_dim); + U(fk("dit_layers" ), m.dit.cfg.layers); + U(fk("dit_interleave" ), m.dit_interleave); + U(fk("attend_text_every_n_blocks"), m.attend_text_every_n); + U(fk("action_horizon" ), m.action_horizon); + U(fk("action_dim" ), m.action_dim); + U(fk("max_state_dim" ), m.max_state_dim); + U(fk("num_inference_timesteps"), m.num_steps); + U(fk("num_timestep_buckets" ), m.num_buckets); + U(fk("max_num_embodiments" ), m.max_embodiments); + U(fk("max_seq_len" ), m.max_seq_len); + + F(fk("vit_ln_eps" ), m.vit.enc.cfg.ln_eps); + F(fk("lm_rms_eps" ), m.lm.cfg.rms_eps); + F(fk("ln_eps" ), m.dit.cfg.ln_eps); + F(fk("norm_out_eps" ), m.dit.cfg.norm_out_eps); + F(fk("vlln_eps" ), m.vlln_eps); + F(fk("connector_ln_eps" ), m.connector_ln_eps); + + if (g.has(fk("lm_rope_theta"))) m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); + + m.vit.enc.cfg.head_dim = m.vit.enc.cfg.hidden/m.vit.enc.cfg.heads; + m.lm.cfg.rope.n_dims = (int) m.lm.cfg.head_dim; + + m.aex.embodiment_id = 20; { const std::string js = g.str(fk("embodiment_id_mapping")); auto lookup = [&](const char * key) -> long { - const std::string k = std::string("\"") + key + "\""; - size_t p = js.find(k); if (p == std::string::npos) return -1; - p = js.find(':', p + k.size()); if (p == std::string::npos) return -1; - return std::strtol(js.c_str() + p + 1, nullptr, 10); + const std::string k = std::string("\"")+key+"\""; + size_t p = js.find(k); + if (p == std::string::npos) return -1; + p = js.find(':', p+k.size()); + if (p == std::string::npos) return -1; + return std::strtol(js.c_str()+p+1, nullptr, 10); }; - long gr1 = lookup("gr1"); if (gr1 >= 0) m.embodiment_id = gr1; + + const long gr1 = lookup("gr1"); + if (gr1 >= 0) m.aex.embodiment_id = gr1; + if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { - char * end = nullptr; long v = std::strtol(e, &end, 10); - if (end && *end == '\0') m.embodiment_id = v; - else { long id = lookup(e); if (id >= 0) m.embodiment_id = id; else std::fprintf(stderr, "vla(gr00tn1d6): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.embodiment_id); } + char * end = nullptr; + const long v = std::strtol(e, &end, 10); + if (end && *end == '\0') { + m.aex.embodiment_id = v; + } else { + const long id = lookup(e); + if (id >= 0) m.aex.embodiment_id = id; + else std::fprintf(stderr, "vla(gr00tn1d6): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); + } } } - if (m.embodiment_id < 0 || m.embodiment_id >= m.max_embodiments) { std::fprintf(stderr, "vla(gr00tn1d6): embodiment id %lld out of range [0,%lld)\n", (long long) m.embodiment_id, (long long) m.max_embodiments); return false; } + if (m.aex.embodiment_id < 0 || m.aex.embodiment_id >= m.max_embodiments) { + std::fprintf(stderr, "vla(gr00tn1d6): embodiment id %lld out of range [0,%lld)\n", + (long long) m.aex.embodiment_id, (long long) m.max_embodiments); + return false; + } // pixel_shuffle_back writes (grid/shuffle)^2 tokens into a buffer sized from // n_img_tokens, so the KV has to agree with the grid it is derived from. - if (m.patch_size <= 0 || m.vit_pixel_shuffle <= 0 || m.image_size % m.patch_size != 0 || - (m.image_size / m.patch_size) % m.vit_pixel_shuffle != 0) { + if (m.patch_size <= 0 || m.vit_pixel_shuffle <= 0 || m.image_size%m.patch_size != 0 || + (m.image_size/m.patch_size)%m.vit_pixel_shuffle != 0) { std::fprintf(stderr, "vla(gr00tn1d6): image %lld / patch %lld / shuffle %lld do not divide evenly\n", (long long) m.image_size, (long long) m.patch_size, (long long) m.vit_pixel_shuffle); return false; } { - const int64_t g2 = (m.image_size / m.patch_size) / m.vit_pixel_shuffle; - if (m.n_img_tokens != g2 * g2) { + const int64_t g2 = (m.image_size/m.patch_size)/m.vit_pixel_shuffle; + if (m.n_img_tokens != g2*g2) { std::fprintf(stderr, "vla(gr00tn1d6): n_img_tokens %lld does not match the %lldx%lld shuffled grid\n", (long long) m.n_img_tokens, (long long) g2, (long long) g2); return false; @@ -271,16 +198,28 @@ bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { } cfg = Config{}; - cfg.n_img = m.n_img_tokens; cfg.n_lang = m.max_seq_len; cfg.n_state = 1; - cfg.n_suffix = m.action_horizon; cfg.max_state_dim = m.max_state_dim; cfg.max_action_dim = m.action_dim; - cfg.real_state_dim = m.max_state_dim; cfg.real_action_dim = m.action_dim; - cfg.hidden = m.lm_hidden; cfg.n_q_heads = m.n_q; cfg.n_kv_heads = m.n_kv; cfg.head_dim = m.lm_head_dim; cfg.n_layers = m.lm_layers; - cfg.num_steps = (int) m.num_steps; cfg.rms_eps = m.lm_rms_eps; - cfg.rope_n_dims = (int) m.lm_head_dim; cfg.rope_mode = GGML_ROPE_TYPE_NEOX; cfg.rope_freq_base = m.lm_rope_base; + cfg.n_img = m.n_img_tokens; + cfg.n_lang = m.max_seq_len; + cfg.n_state = 1; + cfg.n_suffix = m.action_horizon; + cfg.max_state_dim = m.max_state_dim; + cfg.max_action_dim = m.action_dim; + cfg.real_state_dim = m.max_state_dim; + cfg.real_action_dim = m.action_dim; + cfg.hidden = m.lm.cfg.hidden; + cfg.n_q_heads = m.lm.cfg.n_q; + cfg.n_kv_heads = m.lm.cfg.n_kv; + cfg.head_dim = m.lm.cfg.head_dim; + cfg.n_layers = m.lm.cfg.layers; + cfg.num_steps = (int) m.num_steps; + cfg.rms_eps = m.lm.cfg.rms_eps; + cfg.rope_n_dims = (int) m.lm.cfg.head_dim; + cfg.rope_mode = GGML_ROPE_TYPE_NEOX; + cfg.rope_freq_base = m.lm.cfg.rope.freq_base; // Raw output: this arch expects the client to apply the dataset statistics // (see the --stats-json flag in eval/client). - cfg.denormalized = false; - cfg.norm_eps = 1e-8f; + cfg.denormalized = false; + cfg.norm_eps = 1e-8f; return true; } @@ -299,108 +238,58 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, std::printf("vla(gr00tn1d6): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; - if (!g.has("gr00t_n1_6.architecture")) { std::fprintf(stderr, "vla(gr00tn1d6): %s is not a gr00t_n1_6 GGUF\n", ckpt_path.c_str()); return nullptr; } + if (!g.has("gr00t_n1_6.architecture")) { + std::fprintf(stderr, "vla(gr00tn1d6): %s is not a gr00t_n1_6 GGUF\n", ckpt_path.c_str()); + return nullptr; + } if (!load_config(g, *m, m->cfg)) return nullptr; + std::printf("vla(gr00tn1d6): vit=%lldd×%lldL×%lldh (Linear patch embed) pixel_shuffle÷%lld ⇒ n_img_tok=%lld mlp1=LN(%lld)→Linear→GELU→Linear " "lm=Qwen3 %lldd×%lldL (%lldq/%lldkv×%lld) dit=AlternateVLDiT %lldL×%lldh×%lld(inner %lld) attend_text_every_n=%lld in_emb=%lld " "horizon=%lld action_dim=%lld max_state=%lld N_steps=%lld embodiment=%lld resident=%s\n", - (long long) m->vit_hidden, (long long) m->vit_layers, (long long) m->vit_heads, (long long) m->vit_pixel_shuffle, (long long) m->n_img_tokens, (long long) m->mlp_inner, - (long long) m->lm_hidden, (long long) m->lm_layers, (long long) m->n_q, (long long) m->n_kv, (long long) m->lm_head_dim, - (long long) m->dit_layers, (long long) m->dit_heads, (long long) m->dit_head_dim, (long long) m->dit_hidden, (long long) m->attend_text_every_n, (long long) m->in_embed_dim, - (long long) m->action_horizon, (long long) m->action_dim, (long long) m->max_state_dim, (long long) m->num_steps, (long long) m->embodiment_id, + (long long) m->vit.enc.cfg.hidden, (long long) m->vit_layers, (long long) m->vit.enc.cfg.heads, (long long) m->vit_pixel_shuffle, (long long) m->n_img_tokens, (long long) m->mlp_inner, + (long long) m->lm.cfg.hidden, (long long) m->lm.cfg.layers, (long long) m->lm.cfg.n_q, (long long) m->lm.cfg.n_kv, (long long) m->lm.cfg.head_dim, + (long long) m->dit.cfg.layers, (long long) m->dit.cfg.heads, (long long) m->dit.cfg.head_dim, (long long) m->dit.cfg.hidden, (long long) m->attend_text_every_n, (long long) m->in_embed_dim, + (long long) m->action_horizon, (long long) m->action_dim, (long long) m->max_state_dim, (long long) m->num_steps, (long long) m->aex.embodiment_id, m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); - { - const Backend b = backend_init("vla(gr00tn1d6)", m->n_threads); - if (!b.handle) { return nullptr; } - m->backend = b.handle; - } + const Backend b = backend_init("vla(gr00tn1d6)", m->n_threads); + if (!b.handle) return nullptr; + m->backend = b.handle; - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d6): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(gr00tn1d6): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; - - bool ok = true; - - m->vit_patch_w = mk_mm("vit.patch_embd.weight"); - m->vit_patch_b = mk_f32("vit.patch_embd.bias"); - m->vit_pos = mk_f32("vit.pos_embd"); - m->vit_post_ln_w = mk_f32("vit.post_ln.weight"); m->vit_post_ln_b = mk_f32("vit.post_ln.bias"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - ok &= w.ln1w&&w.ln1b&&w.ln2w&&w.ln2b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wfc1&&w.bfc1&&w.Wfc2&&w.bfc2; - } - m->mm_ln_w = mk_f32("mm.ln.weight"); m->mm_ln_b = mk_f32("mm.ln.bias"); - m->mm_fc1_w = mk_mm("mm.fc1.weight"); m->mm_fc1_b = mk_f32("mm.fc1.bias"); - m->mm_fc2_w = mk_mm("mm.fc2.weight"); m->mm_fc2_b = mk_f32("mm.fc2.bias"); - - m->lm_output_norm = mk_f32("vlm.output_norm.weight"); - m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vlm.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->lm[i]; - w.attn_norm=mk_f32(N("attn_norm.weight")); - w.Wq=mk_mm(N("attn_q.weight")); w.Wk=mk_mm(N("attn_k.weight")); w.Wv=mk_mm(N("attn_v.weight")); w.Wo=mk_mm(N("attn_o.weight")); - w.q_norm=mk_f32(N("attn_q_norm.weight")); w.k_norm=mk_f32(N("attn_k_norm.weight")); w.ffn_norm=mk_f32(N("ffn_norm.weight")); - w.Wgate=mk_mm(N("ffn_gate.weight")); w.Wup=mk_mm(N("ffn_up.weight")); w.Wdown=mk_mm(N("ffn_down.weight")); - ok &= w.attn_norm&&w.Wq&&w.Wk&&w.Wv&&w.Wo&&w.q_norm&&w.k_norm&&w.ffn_norm&&w.Wgate&&w.Wup&&w.Wdown; - } + WeightLoader L("gr00tn1d6", g, m->ctx_weights, m->matmul_type); + + m->vit.declare(L, "vit", m->vit_layers, /*patch_embd_is_gemm=*/true); + + m->mm_ln_w = L.f32 ("mm.ln.weight"); + m->mm_ln_b = L.f32 ("mm.ln.bias"); + m->mm_fc1_w = L.gemm("mm.fc1.weight"); + m->mm_fc1_b = L.f32 ("mm.fc1.bias"); + m->mm_fc2_w = L.gemm("mm.fc2.weight"); + m->mm_fc2_b = L.f32 ("mm.fc2.bias"); + + m->lm.declare(L, "vlm"); + + m->vlln_w = L.f32("aex.vlln.weight"); + m->vlln_b = L.f32("aex.vlln.bias"); + + m->aex.declare(L, "aex"); + m->dit.declare(L, "aex.dit"); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; - m->vlln_w=mk_f32("aex.vlln.weight"); m->vlln_b=mk_f32("aex.vlln.bias"); - - m->se_l1W=mk_f32("aex.state_enc.l1.W"); m->se_l1b=mk_f32("aex.state_enc.l1.b"); m->se_l2W=mk_f32("aex.state_enc.l2.W"); m->se_l2b=mk_f32("aex.state_enc.l2.b"); - m->ae_W1W=mk_f32("aex.act_enc.W1.W"); m->ae_W1b=mk_f32("aex.act_enc.W1.b"); m->ae_W2W=mk_f32("aex.act_enc.W2.W"); m->ae_W2b=mk_f32("aex.act_enc.W2.b"); m->ae_W3W=mk_f32("aex.act_enc.W3.W"); m->ae_W3b=mk_f32("aex.act_enc.W3.b"); - m->ad_l1W=mk_f32("aex.act_dec.l1.W"); m->ad_l1b=mk_f32("aex.act_dec.l1.b"); m->ad_l2W=mk_f32("aex.act_dec.l2.W"); m->ad_l2b=mk_f32("aex.act_dec.l2.b"); - m->pos_embd=mk_f32("aex.pos_embd"); - m->te_l1W=mk_mm("aex.dit.time_emb.l1.weight"); m->te_l1b=mk_f32("aex.dit.time_emb.l1.bias"); m->te_l2W=mk_mm("aex.dit.time_emb.l2.weight"); m->te_l2b=mk_f32("aex.dit.time_emb.l2.bias"); - m->dit.resize(m->dit_layers); - for (int64_t i = 0; i < m->dit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "aex.dit.%lld.%s", (long long) i, s); return p; }; - auto & w = m->dit[i]; - w.adaln_w=mk_mm(N("adaln.weight")); w.adaln_b=mk_f32(N("adaln.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.adaln_w&&w.adaln_b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } - m->po1W=mk_mm("aex.dit.proj_out1.weight"); m->po1b=mk_f32("aex.dit.proj_out1.bias"); m->po2W=mk_mm("aex.dit.proj_out2.weight"); m->po2b=mk_f32("aex.dit.proj_out2.bias"); - ok &= m->vit_patch_w&&m->vit_patch_b&&m->vit_pos&&m->vit_post_ln_w&&m->vit_post_ln_b&&m->mm_ln_w&&m->mm_ln_b&&m->mm_fc1_w&&m->mm_fc1_b&&m->mm_fc2_w&&m->mm_fc2_b&&m->lm_output_norm&& - m->vlln_w&&m->vlln_b&&m->se_l1W&&m->se_l1b&&m->se_l2W&&m->se_l2b&&m->ae_W1W&&m->ae_W1b&&m->ae_W2W&&m->ae_W2b&&m->ae_W3W&&m->ae_W3b&& - m->ad_l1W&&m->ad_l1b&&m->ad_l2W&&m->ad_l2b&&m->pos_embd&&m->te_l1W&&m->te_l1b&&m->te_l2W&&m->te_l2b&&m->po1W&&m->po1b&&m->po2W&&m->po2b; - if (!ok) { std::fprintf(stderr, "vla(gr00tn1d6): weight tensor setup failed\n"); return nullptr; } - - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(gr00tn1d6): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(gr00tn1d6): failed to load %s (%zu vs %zu bytes)\n", ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } std::printf("vla(gr00tn1d6): weights resident in %.2f GiB (%s) - incl. SigLIP2 vision tower; embodiment id %lld\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", (long long) m->embodiment_id); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), + m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", (long long) m->aex.embodiment_id); return m; } @@ -408,230 +297,213 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { const auto t0 = std::chrono::steady_clock::now(); stats = Stats{}; - const int64_t H = lm_hidden, K = n_img_tokens, E = in_embed_dim; - const int64_t grid = image_size / patch_size; - const int64_t r = vit_pixel_shuffle; - const int64_t n_patches = grid * grid; - const int64_t patch_dim = 3 * patch_size * patch_size; - const int64_t c4 = vit_hidden * r * r; - const int64_t Nsa = 1 + action_horizon; + const int64_t H = lm.cfg.hidden; + const int64_t K = n_img_tokens; + const int64_t E = in_embed_dim; + const int64_t grid = image_size/patch_size; + const int64_t r = vit_pixel_shuffle; + const int64_t n_patches = grid*grid; + const int64_t patch_dim = 3*patch_size*patch_size; + const int64_t c4 = vit.enc.cfg.hidden*r*r; + const int64_t AD = action_dim; + const int64_t AH = action_horizon; + const int64_t Nsa = 1+AH; int64_t n_views = 0; std::vector img_emb_host; const float * img_emb_ptr = nullptr; + if (in.precomputed_img_emb && in.n_img_views > 0) { - n_views = in.n_img_views; + n_views = in.n_img_views; img_emb_ptr = in.precomputed_img_emb; } else if (in.images && in.n_images > 0) { n_views = in.n_images; - img_emb_host.assign((size_t) n_views * K * H, 0.0f); + img_emb_host.assign((size_t) n_views*K*H, 0.0f); - ggml_context * VC = vision_scratch.reset((size_t) 64 * 1024 * 1024); + ggml_context * VC = vision_scratch.reset((size_t) 64*1024*1024); if (!VC) { std::fprintf(stderr, "vla(gr00tn1d6): ggml_init(vision ctx A) failed\n"); return {}; } - ggml_tensor * t_patches = ggml_new_tensor_3d(VC, GGML_TYPE_F32, patch_dim, n_patches, n_views); ggml_set_input(t_patches); - ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit_patch_w, t_patches), vit_patch_b), vit_pos); - for (int64_t i = 0; i < vit_layers; ++i) h = build_siglip_layer(VC, vit[i], h, n_patches, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps); - ggml_tensor * post_ln = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit_post_ln_w), vit_post_ln_b); + + ggml_tensor * t_patches = ggml_new_tensor_3d(VC, GGML_TYPE_F32, patch_dim, n_patches, n_views); + ggml_set_input(t_patches); + ggml_tensor * post_ln = vit.build(VC, vit.embed_patches(VC, t_patches), n_patches, n_views); ggml_set_output(post_ln); + ggml_cgraph * vgA = ggml_new_graph_custom(VC, 8192, false); ggml_build_forward_expand(vgA, post_ln); if (!vision_scratch.alloc(backend, vgA)) { std::fprintf(stderr, "vla(gr00tn1d6): vision gallocr A alloc failed\n"); return {}; } - ggml_context * MC = merge_scratch.reset((size_t) 16 * 1024 * 1024); + ggml_context * MC = merge_scratch.reset((size_t) 16*1024*1024); if (!MC) { std::fprintf(stderr, "vla(gr00tn1d6): ggml_init(vision ctx B) failed\n"); return {}; } - ggml_tensor * t_shuf = ggml_new_tensor_3d(MC, GGML_TYPE_F32, c4, K, n_views); ggml_set_input(t_shuf); - ggml_tensor * mln = ggml_add(MC, ggml_mul(MC, ggml_norm(MC, t_shuf, connector_ln_eps), mm_ln_w), mm_ln_b); - ggml_tensor * mz1 = ggml_add(MC, ggml_mul_mat(MC, mm_fc1_w, mln), mm_fc1_b); - ggml_tensor * vit_embeds = ggml_add(MC, ggml_mul_mat(MC, mm_fc2_w, ggml_gelu_erf(MC, mz1)), mm_fc2_b); + + ggml_tensor * t_shuf = ggml_new_tensor_3d(MC, GGML_TYPE_F32, c4, K, n_views); + ggml_set_input(t_shuf); + ggml_tensor * mln = layer_norm(MC, t_shuf, mm_ln_w, mm_ln_b, connector_ln_eps); + ggml_tensor * vit_embeds = ffn_gelu_erf(MC, mm_fc1_w, mm_fc1_b, mm_fc2_w, mm_fc2_b, mln); ggml_set_output(vit_embeds); + ggml_cgraph * vgB = ggml_new_graph(MC); ggml_build_forward_expand(vgB, vit_embeds); if (!merge_scratch.alloc(backend, vgB)) { std::fprintf(stderr, "vla(gr00tn1d6): vision gallocr B alloc failed\n"); return {}; } const auto tv0 = std::chrono::steady_clock::now(); - std::vector patches, - patches_all((size_t) patch_dim * n_patches * n_views), - post_ln_host((size_t) vit_hidden * n_patches * n_views), - shuf_host((size_t) c4 * K * n_views); + std::vector patches; + std::vector patches_all((size_t) patch_dim*n_patches*n_views); + std::vector post_ln_host((size_t) vit.enc.cfg.hidden*n_patches*n_views); + std::vector shuf_host((size_t) c4*K*n_views); + bool vok = true; for (int64_t v = 0; v < n_views && vok; ++v) { - if (!preprocess_image_patches(in.images[v], image_size, patch_size, patches)) { vok = false; break; } - std::memcpy(patches_all.data() + (size_t) v * patch_dim * n_patches, patches.data(), patches.size() * sizeof(float)); + if (!preprocess_image_patches("gr00tn1d6", in.images[v], image_size, patch_size, patches)) { vok = false; break; } + std::memcpy(patches_all.data()+(size_t) v*patch_dim*n_patches, patches.data(), patches.size()*sizeof(float)); } if (vok) { ggml_backend_tensor_set(t_patches, patches_all.data(), 0, ggml_nbytes(t_patches)); - if (ggml_backend_graph_compute(backend, vgA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute A failed\n"); vok = false; } + if (ggml_backend_graph_compute(backend, vgA) != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d6): vision compute A failed\n"); + vok = false; + } } if (vok) { ggml_backend_tensor_get(post_ln, post_ln_host.data(), 0, ggml_nbytes(post_ln)); for (int64_t v = 0; v < n_views; ++v) - pixel_shuffle_back(post_ln_host.data() + (size_t) v * vit_hidden * n_patches, grid, vit_hidden, r, shuf_host.data() + (size_t) v * c4 * K); + pixel_shuffle_back(post_ln_host.data()+(size_t) v*vit.enc.cfg.hidden*n_patches, grid, vit.enc.cfg.hidden, r, + shuf_host.data()+(size_t) v*c4*K); + ggml_backend_tensor_set(t_shuf, shuf_host.data(), 0, ggml_nbytes(t_shuf)); - if (ggml_backend_graph_compute(backend, vgB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute B failed\n"); vok = false; } + if (ggml_backend_graph_compute(backend, vgB) != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d6): vision compute B failed\n"); + vok = false; + } } if (vok) ggml_backend_tensor_get(vit_embeds, img_emb_host.data(), 0, ggml_nbytes(vit_embeds)); - stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); if (!vok) return {}; img_emb_ptr = img_emb_host.data(); } else { - std::fprintf(stderr, "vla(gr00tn1d6): no images and no precomputed_img_emb in the request\n"); return {}; - } - const int64_t n_img = n_views * K; - - std::vector input_ids; - int64_t n_img_slots = 0; - for (int j = 0; j < in.n_lang; ++j) if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_img_slots; - if (n_img_slots == n_img) { - input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); - } else if (n_img_slots == 0) { - input_ids.reserve(n_img + in.n_lang); - for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); - } else { - std::fprintf(stderr, "vla(gr00tn1d6): lang_tokens has %lld image-token slots but n_img=%lld; expected 0 (v1 fallback) or %lld (chat-template path)\n", - (long long) n_img_slots, (long long) n_img, (long long) n_img); + std::fprintf(stderr, "vla(gr00tn1d6): no images and no precomputed_img_emb in the request\n"); return {}; } - const int64_t SEQ = (int64_t) input_ids.size(); - if (SEQ > max_seq_len) { std::fprintf(stderr, "vla(gr00tn1d6): prompt too long (%lld > %lld)\n", (long long) SEQ, (long long) max_seq_len); return {}; } - - std::vector inputs_embeds((size_t) SEQ * H); - if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), H)) return {}; - { int64_t k = 0; - for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { - if (k >= n_img) { std::fprintf(stderr, "vla(gr00tn1d6): more tokens than ViT embeds\n"); return {}; } - std::memcpy(inputs_embeds.data() + p * H, img_emb_ptr + k * H, H * sizeof(float)); ++k; - } - } + const int64_t n_img = n_views*K; - std::vector image_pos_idx, text_pos_idx; - image_pos_idx.reserve((size_t) n_img); text_pos_idx.reserve((size_t) (SEQ - n_img)); - for (int64_t p = 0; p < SEQ; ++p) { - if (input_ids[p] == (int32_t) image_token_index) image_pos_idx.push_back((int32_t) p); - else text_pos_idx.push_back((int32_t) p); - } - const int64_t SEQ_TXT = (int64_t) text_pos_idx.size(); - if ((int64_t) image_pos_idx.size() != n_img) { - std::fprintf(stderr, "vla(gr00tn1d6): internal: built %zu image positions, expected %lld\n", image_pos_idx.size(), (long long) n_img); return {}; - } + Prompt prompt; + if (!build_prompt("gr00tn1d6", in, n_img, (int32_t) image_token_index, max_seq_len, prompt)) return {}; + const int64_t SEQ = prompt.len(); + const int64_t SEQ_TXT = prompt.n_text(); - const int64_t AD = action_dim, AH = action_horizon; - std::vector x_init((size_t) AH * AD); - if (in.noise) std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); - else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); for (auto & v : x_init) v = nd(rng); } + std::vector inputs_embeds; + if (!fetch_embeds("gr00tn1d6", io, prompt, img_emb_ptr, H, inputs_embeds)) return {}; + + std::vector x_init; + init_noise(in, (size_t) AH*AD, x_init); // LM + DiT graph depends only on the sequence split and step count. const MainKey mkey{ SEQ, n_img, SEQ_TXT, num_steps }; - const bool built = main_graph.ensure(backend, mkey, (size_t) 256 * 1024 * 1024, + const bool built = main_graph.ensure(backend, mkey, (size_t) 256*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { - ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); - ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ); ggml_set_input(t_pos); - ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); - ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1);ggml_set_input(t_state); - ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); - - ggml_tensor * t_img_idx = ggml_new_tensor_1d(C, GGML_TYPE_I32, n_img); ggml_set_input(t_img_idx); - ggml_tensor * t_txt_idx = (SEQ_TXT > 0) ? ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ_TXT) : nullptr; - if (t_txt_idx) ggml_set_input(t_txt_idx); - std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { - t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); - t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, 256); ggml_set_input(t_tproj[s]); - } - - ggml_tensor * h = t_embeds; - for (int64_t i = 0; i < lm_layers; ++i) h = build_qwen3_layer(C, *this, lm[i], h, t_pos, t_lmmask, SEQ); - ggml_tensor * eagle = ggml_mul(C, ggml_rms_norm(C, h, lm_rms_eps), lm_output_norm); + ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); + ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ); ggml_set_input(t_pos); + ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); + ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1); ggml_set_input(t_state); + ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); + + ggml_tensor * t_img_idx = ggml_new_tensor_1d(C, GGML_TYPE_I32, n_img); + ggml_set_input(t_img_idx); + ggml_tensor * t_txt_idx = (SEQ_TXT > 0) ? ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ_TXT) : nullptr; + if (t_txt_idx) ggml_set_input(t_txt_idx); + + std::vector t_tau(num_steps), t_tproj(num_steps); + for (int64_t s = 0; s < num_steps; ++s) { + t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); + t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, 256); ggml_set_input(t_tproj[s]); + } - ggml_tensor * vl_embs = ggml_add(C, ggml_mul(C, ggml_norm(C, eagle, vlln_eps), vlln_w), vlln_b); + ggml_tensor * eagle = lm.build(C, t_embeds, t_pos, t_lmmask, SEQ); + ggml_tensor * vl_embs = layer_norm(C, eagle, vlln_w, vlln_b, vlln_eps); + ggml_tensor * vl_img = ggml_get_rows(C, vl_embs, t_img_idx); + ggml_tensor * vl_txt = t_txt_idx ? ggml_get_rows(C, vl_embs, t_txt_idx) : vl_img; - ggml_tensor * vl_img = ggml_get_rows(C, vl_embs, t_img_idx); - ggml_tensor * vl_txt = (t_txt_idx ? ggml_get_rows(C, vl_embs, t_txt_idx) : vl_img); + ggml_tensor * state_features = aex.encode_state(C, t_state); - ggml_tensor * state_features = cat_linear(C, se_l2W, se_l2b, embodiment_id, ggml_relu(C, cat_linear(C, se_l1W, se_l1b, embodiment_id, t_state))); + const float dt = 1.0f/(float) num_steps; + const int64_t every2 = 2*attend_text_every_n; - const float dt = 1.0f / (float) num_steps; - const int64_t every2 = 2 * attend_text_every_n; + std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); + for (int64_t i = 0; i < dit.cfg.layers; ++i) { + if (dit_interleave && (i%2 == 1)) continue; + dit.kv(C, dit.blk[i], (i%every2 == 0) ? vl_txt : vl_img, &Kc[i], &Vc[i]); + } - std::vector Kc(dit_layers, nullptr), Vc(dit_layers, nullptr); - for (int64_t i = 0; i < dit_layers; ++i) { - if (dit_interleave && (i % 2 == 1)) continue; - ggml_tensor * enc = (i % every2 == 0) ? vl_txt : vl_img; - dit_kv(C, *this, dit[i], enc, &Kc[i], &Vc[i]); - } - ggml_tensor * actions = t_x0; - for (int64_t s = 0; s < num_steps; ++s) { + ggml_tensor * actions = t_x0; + for (int64_t s = 0; s < num_steps; ++s) { + ggml_tensor * temb = dit.time_emb(C, t_tproj[s]); + ggml_tensor * af = aex.encode_action(C, actions, t_tau[s], E, AH); + ggml_tensor * hh = ggml_concat(C, state_features, af, 1); + + for (int64_t i = 0; i < dit.cfg.layers; ++i) { + ggml_tensor * enc; + if (dit_interleave && (i%2 == 1)) enc = nullptr; + else if (i%every2 == 0) enc = vl_txt; + else enc = vl_img; + hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); + } + + ggml_tensor * pred = aex.decode(C, dit.proj_out(C, hh, temb)); + ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t)(Nsa-AH)*pred->nb[1])); + actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); + } + ggml_set_name(actions, "action_pred"); + ggml_set_output(actions); - ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, te_l1W, t_tproj[s]), te_l1b))), te_l2b); + gio.t_embeds=t_embeds; gio.t_pos=t_pos; gio.t_lmmask=t_lmmask; gio.t_state=t_state; gio.t_x0=t_x0; + gio.t_img_idx=t_img_idx; gio.t_txt_idx=t_txt_idx; gio.t_tau=t_tau; gio.t_tproj=t_tproj; gio.actions=actions; - ggml_tensor * a_emb = cat_linear(C, ae_W1W, ae_W1b, embodiment_id, actions); - ggml_tensor * x_w2 = ggml_silu(C, cat_linear(C, ae_W2W, ae_W2b, embodiment_id, ggml_concat(C, a_emb, t_tau[s], 0))); - ggml_tensor * af = ggml_add(C, cat_linear(C, ae_W3W, ae_W3b, embodiment_id, x_w2), ggml_view_2d(C, pos_embd, E, AH, pos_embd->nb[1], 0)); + ggml_cgraph * gf = ggml_new_graph_custom(C, 65536, false); + ggml_build_forward_expand(gf, actions); + return gf; + }); + if (!built) { std::fprintf(stderr, "vla(gr00tn1d6): main graph build failed\n"); return {}; } - ggml_tensor * sa = ggml_concat(C, state_features, af, 1); + MainIO & gio = main_graph.io(); - ggml_tensor * hh = sa; - for (int64_t i = 0; i < dit_layers; ++i) { - ggml_tensor * enc; - if (dit_interleave && (i % 2 == 1)) enc = nullptr; - else if (i % every2 == 0) enc = vl_txt; - else enc = vl_img; - hh = build_dit_block(C, *this, dit[i], hh, temb, enc, Kc[i], Vc[i]); - } + ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); - ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, po1W, ggml_silu(C, temb)), po1b); - ggml_tensor * sh = ggml_view_1d(C, po, dit_hidden, 0), * sc = ggml_view_1d(C, po, dit_hidden, (size_t) dit_hidden * sizeof(float)); - ggml_tensor * hn = ggml_norm(C, hh, norm_out_eps); - ggml_tensor * h_mod = ggml_add(C, ggml_add(C, hn, ggml_mul(C, hn, sc)), sh); - ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, po2W, h_mod), po2b); + std::vector pp(SEQ); + for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; + ggml_backend_tensor_set(gio.t_pos, pp.data(), 0, ggml_nbytes(gio.t_pos)); - ggml_tensor * pred = cat_linear(C, ad_l2W, ad_l2b, embodiment_id, ggml_relu(C, cat_linear(C, ad_l1W, ad_l1b, embodiment_id, model_output))); - ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t) (Nsa - AH) * pred->nb[1])); - actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); - } - ggml_set_name(actions, "action_pred"); ggml_set_output(actions); + std::vector mask; + build_causal_mask(SEQ, mask); + ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); - gio.t_embeds=t_embeds; gio.t_pos=t_pos; gio.t_lmmask=t_lmmask; gio.t_state=t_state; gio.t_x0=t_x0; - gio.t_img_idx=t_img_idx; gio.t_txt_idx=t_txt_idx; gio.t_tau=t_tau; gio.t_tproj=t_tproj; gio.actions=actions; + std::vector st(max_state_dim, 0.0f); + for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; + ggml_backend_tensor_set(gio.t_state, st.data(), 0, ggml_nbytes(gio.t_state)); - ggml_cgraph * gf = ggml_new_graph_custom(C, 65536, false); - ggml_build_forward_expand(gf, actions); - return gf; - }); - if (!built) { std::fprintf(stderr, "vla(gr00tn1d6): main graph build failed\n"); return {}; } + ggml_backend_tensor_set(gio.t_x0, x_init.data(), 0, ggml_nbytes(gio.t_x0)); + ggml_backend_tensor_set(gio.t_img_idx, prompt.image_pos.data(), 0, ggml_nbytes(gio.t_img_idx)); + if (gio.t_txt_idx) ggml_backend_tensor_set(gio.t_txt_idx, prompt.text_pos.data(), 0, ggml_nbytes(gio.t_txt_idx)); - MainIO & gio = main_graph.io(); - ggml_cgraph * gf = main_graph.graph(); - ggml_tensor * t_embeds = gio.t_embeds, * t_pos = gio.t_pos, * t_lmmask = gio.t_lmmask; - ggml_tensor * t_state = gio.t_state, * t_x0 = gio.t_x0; - ggml_tensor * t_img_idx = gio.t_img_idx, * t_txt_idx = gio.t_txt_idx, * actions = gio.actions; - std::vector & t_tau = gio.t_tau; std::vector & t_tproj = gio.t_tproj; - - ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); - { std::vector pp(SEQ); for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } - { std::vector mk((size_t) SEQ * SEQ); const float NEG = -std::numeric_limits::infinity(); - for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ + kv] = (kv <= q) ? 0.0f : NEG; - ggml_backend_tensor_set(t_lmmask, mk.data(), 0, ggml_nbytes(t_lmmask)); } - { std::vector st(max_state_dim, 0.0f); for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); } - ggml_backend_tensor_set(t_x0, x_init.data(), 0, ggml_nbytes(t_x0)); - ggml_backend_tensor_set(t_img_idx, image_pos_idx.data(), 0, ggml_nbytes(t_img_idx)); - if (t_txt_idx) ggml_backend_tensor_set(t_txt_idx, text_pos_idx.data(), 0, ggml_nbytes(t_txt_idx)); for (int64_t s = 0; s < num_steps; ++s) { - const int64_t bucket = (int64_t) ((double) s / (double) num_steps * (double) num_buckets); - std::vector tau, tpr; action_sinusoid(bucket, E, AH, tau); timesteps_proj(bucket, tpr); - ggml_backend_tensor_set(t_tau[s], tau.data(), 0, ggml_nbytes(t_tau[s])); - ggml_backend_tensor_set(t_tproj[s], tpr.data(), 0, ggml_nbytes(t_tproj[s])); + const int64_t bucket = (int64_t) ((double) s/(double) num_steps*(double) num_buckets); + std::vector tau, tpr; + action_sinusoid(bucket, E, AH, tau); + timesteps_proj(bucket, tpr); + ggml_backend_tensor_set(gio.t_tau[s], tau.data(), 0, ggml_nbytes(gio.t_tau[s])); + ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } const auto tc0 = std::chrono::steady_clock::now(); - const ggml_status st = ggml_backend_graph_compute(backend, gf); + const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); - if (st != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): graph compute failed (%d)\n", (int) st); return {}; } - stats.ms_inference = std::chrono::duration(tc1 - tc0).count(); + if (status != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d6): graph compute failed (%d)\n", (int) status); + return {}; + } + stats.ms_inference = std::chrono::duration(tc1-tc0).count(); - std::vector out((size_t) AH * AD); - ggml_backend_tensor_get(actions, out.data(), 0, out.size() * sizeof(float)); - stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + std::vector out((size_t) AH*AD); + ggml_backend_tensor_get(gio.actions, out.data(), 0, out.size()*sizeof(float)); + stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now()-t0).count(); return out; } diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 227399d..fbd6b3b 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -20,10 +20,10 @@ #include "ggml-backend.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "models/dit_common.h" -#include "models/qwen3vl_vit.h" +#include "modules/qwen3vl_vit.h" #include "env_flag.h" #include diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index 13e056e..19c6b08 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -14,16 +14,16 @@ #include "arch.h" #include "model.h" -#include "vision_common.h" -#include "models/dual_tower.h" +#include "modules/preprocess.h" +#include "modules/dual_tower.h" #include "ggml.h" #include "ggml-cpu.h" #include "ggml-backend.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "env_flag.h" #include diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 1396de3..027b57a 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -21,11 +21,11 @@ #include "ggml-alloc.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "models/dit_common.h" -#include "models/vision_common.h" -#include "models/act_dtype.h" +#include "modules/preprocess.h" +#include "act_dtype.h" #include "cuda/vla_cuda_ops.h" #include "env_flag.h" diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index d51eee0..ccc9114 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -21,10 +21,10 @@ #include "ggml-alloc.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "models/dit_common.h" -#include "models/vision_common.h" +#include "modules/preprocess.h" #include "env_flag.h" #include diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index a921d74..c06ce17 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -17,9 +17,9 @@ #include "arch.h" #include "model.h" -#include "vision_common.h" +#include "modules/preprocess.h" #include "scratch_ctx.h" -#include "dit_common.h" +#include "models/dit_common.h" #include "ggml.h" #include "ggml-backend.h" diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 2f905a4..82e74d0 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -14,16 +14,16 @@ #include "arch.h" #include "model.h" -#include "vision_common.h" -#include "models/dual_tower.h" +#include "modules/preprocess.h" +#include "modules/dual_tower.h" #include "ggml.h" #include "ggml-cpu.h" #include "ggml-backend.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "models/dit_common.h" #include "env_flag.h" diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 2f32eef..9bee501 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -20,10 +20,10 @@ #include "ggml-backend.h" #include "backend.h" #include "gguf.h" -#include "models/gguf_reader.h" -#include "models/scratch_ctx.h" +#include "gguf_reader.h" +#include "scratch_ctx.h" #include "models/dit_common.h" -#include "models/qwen3vl_vit.h" +#include "modules/qwen3vl_vit.h" #include "env_flag.h" #include diff --git a/src/modules/action_expert.cpp b/src/modules/action_expert.cpp new file mode 100644 index 0000000..0f21fca --- /dev/null +++ b/src/modules/action_expert.cpp @@ -0,0 +1,60 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/action_expert.h" + +#include "layers/linear.h" + +namespace vla { + +void ActionExpert::declare(WeightLoader & L, const char * prefix) { + se_l1W = L.f32("%s.state_enc.l1.W", prefix); + se_l1b = L.f32("%s.state_enc.l1.b", prefix); + se_l2W = L.f32("%s.state_enc.l2.W", prefix); + se_l2b = L.f32("%s.state_enc.l2.b", prefix); + + ae_W1W = L.f32("%s.act_enc.W1.W", prefix); + ae_W1b = L.f32("%s.act_enc.W1.b", prefix); + ae_W2W = L.f32("%s.act_enc.W2.W", prefix); + ae_W2b = L.f32("%s.act_enc.W2.b", prefix); + ae_W3W = L.f32("%s.act_enc.W3.W", prefix); + ae_W3b = L.f32("%s.act_enc.W3.b", prefix); + + ad_l1W = L.f32("%s.act_dec.l1.W", prefix); + ad_l1b = L.f32("%s.act_dec.l1.b", prefix); + ad_l2W = L.f32("%s.act_dec.l2.W", prefix); + ad_l2b = L.f32("%s.act_dec.l2.b", prefix); + + pos_embd = L.f32("%s.pos_embd", prefix); +} + +ggml_tensor * ActionExpert::encode_state(ggml_context * C, ggml_tensor * state) const { + ggml_tensor * h = ggml_relu(C, cat_linear(C, se_l1W, se_l1b, embodiment_id, state)); + return cat_linear(C, se_l2W, se_l2b, embodiment_id, h); +} + +ggml_tensor * ActionExpert::encode_action(ggml_context * C, ggml_tensor * actions, ggml_tensor * tau, + int64_t embed_dim, int64_t horizon) const { + ggml_tensor * a_emb = cat_linear(C, ae_W1W, ae_W1b, embodiment_id, actions); + ggml_tensor * x_w2 = ggml_silu(C, cat_linear(C, ae_W2W, ae_W2b, embodiment_id, ggml_concat(C, a_emb, tau, 0))); + ggml_tensor * pos = ggml_view_2d(C, pos_embd, embed_dim, horizon, pos_embd->nb[1], 0); + return ggml_add(C, cat_linear(C, ae_W3W, ae_W3b, embodiment_id, x_w2), pos); +} + +ggml_tensor * ActionExpert::decode(ggml_context * C, ggml_tensor * model_out) const { + ggml_tensor * h = ggml_relu(C, cat_linear(C, ad_l1W, ad_l1b, embodiment_id, model_out)); + return cat_linear(C, ad_l2W, ad_l2b, embodiment_id, h); +} + +} diff --git a/src/modules/action_expert.h b/src/modules/action_expert.h new file mode 100644 index 0000000..62c558f --- /dev/null +++ b/src/modules/action_expert.h @@ -0,0 +1,50 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GR00T action expert: the per-embodiment MLPs that lift state and noisy +// actions into the DiT token stream and project the DiT output back to action +// space. Every projection is a cat_linear row selected by embodiment_id. + +#pragma once + +#include "loader.h" + +#include "ggml.h" + +#include + +namespace vla { + +struct ActionExpert { + ggml_tensor *se_l1W = nullptr, *se_l1b = nullptr, *se_l2W = nullptr, *se_l2b = nullptr; + ggml_tensor *ae_W1W = nullptr, *ae_W1b = nullptr, *ae_W2W = nullptr, *ae_W2b = nullptr; + ggml_tensor *ae_W3W = nullptr, *ae_W3b = nullptr; + ggml_tensor *ad_l1W = nullptr, *ad_l1b = nullptr, *ad_l2W = nullptr, *ad_l2b = nullptr; + ggml_tensor *pos_embd = nullptr; + + int64_t embodiment_id = 0; + + // Reads ".state_enc.*", ".act_enc.*", ".act_dec.*" + // and ".pos_embd". + void declare(WeightLoader & L, const char * prefix); + + ggml_tensor * encode_state(ggml_context * C, ggml_tensor * state) const; + + ggml_tensor * encode_action(ggml_context * C, ggml_tensor * actions, ggml_tensor * tau, + int64_t embed_dim, int64_t horizon) const; + + ggml_tensor * decode(ggml_context * C, ggml_tensor * model_out) const; +}; + +} diff --git a/src/modules/dit_head.cpp b/src/modules/dit_head.cpp new file mode 100644 index 0000000..01f8405 --- /dev/null +++ b/src/modules/dit_head.cpp @@ -0,0 +1,141 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/dit_head.h" + +#include "layers/attn.h" +#include "layers/ffn.h" +#include "layers/linear.h" +#include "layers/norm.h" + +#include +#include + +namespace vla { + +void DitHead::declare(WeightLoader & L, const char * prefix, bool fuse_qkv, bool interleave) { + te_l1W = L.gemm("%s.time_emb.l1.weight", prefix); + te_l1b = L.f32 ("%s.time_emb.l1.bias", prefix); + te_l2W = L.gemm("%s.time_emb.l2.weight", prefix); + te_l2b = L.f32 ("%s.time_emb.l2.bias", prefix); + + blk.resize(cfg.layers); + for (int64_t i=0; ine[1]; + + if (w.Wkv) { + ggml_tensor * kvp = linear(C, w.Wkv, w.bkv, src); + *K_out = ggml_cont(C, ggml_permute(C, head_view(C, kvp, hd, heads, Tkv, cfg.hidden, 2, 0), 0, 2, 1, 3)); + *V_out = ggml_cont(C, ggml_permute(C, head_view(C, kvp, hd, heads, Tkv, cfg.hidden, 2, 1), 1, 2, 0, 3)); + return; + } + *K_out = to_heads (C, linear(C, w.Wk, w.bk, src), hd, heads, Tkv); + *V_out = to_heads_v(C, linear(C, w.Wv, w.bv, src), hd, heads, Tkv); +} + +ggml_tensor * DitHead::block(ggml_context * C, const DitLayerW & w, ggml_tensor * h, ggml_tensor * temb, + ggml_tensor * enc, ggml_tensor * K_pre, ggml_tensor * V_pre) const { + const int64_t hd = cfg.head_dim; + const int64_t heads = cfg.heads; + const int64_t dim = cfg.hidden; + const int64_t Tk = h->ne[1]; + const float scale = 1.0f/std::sqrt((float)hd); + + ggml_tensor * n = adaln(C, h, temb, w.adaln_w, w.adaln_b, dim, cfg.ln_eps); + ggml_tensor *Q, *K, *V; + if (!enc && w.Wqkv) { + ggml_tensor * qkv = linear(C, w.Wqkv, w.bqkv, n); + Q = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 0), 0, 2, 1, 3)); + K = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 1), 0, 2, 1, 3)); + V = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 2), 1, 2, 0, 3)); + } else { + Q = to_heads(C, linear(C, w.Wq, w.bq, n), hd, heads, Tk); + if (K_pre) { K = K_pre; V = V_pre; } + else { kv(C, w, enc ? enc : n, &K, &V); } + } + + ggml_tensor * att = attention(C, Q, K, V, nullptr, scale, dim, Tk); + ggml_tensor * h1 = ggml_add(C, h, linear(C, w.Wo, w.bo, att)); + ggml_tensor * n3 = ggml_norm(C, h1, cfg.ln_eps); + return ggml_add(C, h1, ffn_gelu(C, w.Wff0, w.bff0, w.Wff2, w.bff2, n3)); +} + +ggml_tensor * DitHead::time_emb(ggml_context * C, ggml_tensor * tproj) const { + return linear(C, te_l2W, te_l2b, ggml_silu(C, linear(C, te_l1W, te_l1b, tproj))); +} + +ggml_tensor * DitHead::proj_out(ggml_context * C, ggml_tensor * h, ggml_tensor * temb) const { + ggml_tensor * po = linear(C, po1W, po1b, ggml_silu(C, temb)); + ggml_tensor * sh = ggml_view_1d(C, po, cfg.hidden, 0); + ggml_tensor * sc = ggml_view_1d(C, po, cfg.hidden, (size_t)cfg.hidden*sizeof(float)); + + ggml_tensor * hn = ggml_norm(C, h, cfg.norm_out_eps); + ggml_tensor * h_mod = ggml_add(C, ggml_add(C, hn, ggml_mul(C, hn, sc)), sh); + return linear(C, po2W, po2b, h_mod); +} + +} diff --git a/src/modules/dit_head.h b/src/modules/dit_head.h new file mode 100644 index 0000000..d58a2fc --- /dev/null +++ b/src/modules/dit_head.h @@ -0,0 +1,76 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// DiT action head: adaLN-conditioned blocks that alternate self-attention with +// cross-attention into the vision-language embeddings. Shared by GR00T +// N1.5/N1.6/N1.7 and VLA-JEPA. +// +// A block cross-attends when `enc` is non-null and self-attends otherwise. The +// per-layer choice is the model's, since each arch interleaves differently. +// Cross-attention K/V depend only on `enc`, so a model hoists them out of the +// solver loop and passes them back through K_pre/V_pre. + +#pragma once + +#include "loader.h" + +#include "ggml.h" + +#include +#include + +namespace vla { + +struct DitLayerW { + ggml_tensor *adaln_w, *adaln_b; + ggml_tensor *Wq, *bq, *Wk, *bk, *Wv, *bv, *Wo, *bo; + ggml_tensor *Wff0, *bff0, *Wff2, *bff2; + + // N1.7 ships self-attention QKV and cross-attention KV pre-fused. + ggml_tensor *Wqkv = nullptr, *bqkv = nullptr, *Wkv = nullptr, *bkv = nullptr; +}; + +struct DitCfg { + int64_t hidden = 1536; + int64_t heads = 32; + int64_t head_dim = 48; + int64_t layers = 16; + float ln_eps = 1e-5f; + float norm_out_eps = 1e-6f; +}; + +struct DitHead { + DitCfg cfg; + std::vector blk; + ggml_tensor *te_l1W = nullptr, *te_l1b = nullptr, *te_l2W = nullptr, *te_l2b = nullptr; + ggml_tensor *po1W = nullptr, *po1b = nullptr, *po2W = nullptr, *po2b = nullptr; + + // Reads "..*", ".time_emb.*" and ".proj_out*". + // fuse_qkv builds N1.7's synthetic Wqkv (self blocks) and Wkv (cross blocks) + // by concatenating the split projections at load. + void declare(WeightLoader & L, const char * prefix, bool fuse_qkv = false, bool interleave = false); + + void kv(ggml_context * C, const DitLayerW & w, ggml_tensor * src, + ggml_tensor ** K_out, ggml_tensor ** V_out) const; + + ggml_tensor * block(ggml_context * C, const DitLayerW & w, ggml_tensor * h, ggml_tensor * temb, + ggml_tensor * enc, ggml_tensor * K_pre = nullptr, ggml_tensor * V_pre = nullptr) const; + + ggml_tensor * time_emb(ggml_context * C, ggml_tensor * tproj) const; + + // Final (shift, scale) adaLN and output projection. + ggml_tensor * proj_out(ggml_context * C, ggml_tensor * h, ggml_tensor * temb) const; +}; + +} diff --git a/src/models/dual_tower.h b/src/modules/dual_tower.h similarity index 100% rename from src/models/dual_tower.h rename to src/modules/dual_tower.h diff --git a/src/modules/encoder.cpp b/src/modules/encoder.cpp new file mode 100644 index 0000000..6c31ac1 --- /dev/null +++ b/src/modules/encoder.cpp @@ -0,0 +1,79 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/encoder.h" + +#include "layers/attn.h" +#include "layers/ffn.h" +#include "layers/linear.h" +#include "layers/norm.h" + +#include + +namespace vla { + +void EncStack::declare(WeightLoader & L, const char * prefix, int64_t layers, const EncNames & n) { + blk.resize(layers); + + for (int64_t i=0; i +#include + +namespace vla { + +struct EncBlockW { + ggml_tensor *ln1w, *ln1b, *ln2w, *ln2b; + ggml_tensor *Wq, *bq, *Wk, *bk, *Wv, *bv, *Wo, *bo; + ggml_tensor *Wfc1, *bfc1, *Wfc2, *bfc2; +}; + +struct EncNames { + const char * ln1 = "ln1"; + const char * ln2 = "ln2"; + const char * fc1 = "fc1"; + const char * fc2 = "fc2"; +}; + +struct EncCfg { + int64_t hidden = 0; + int64_t heads = 0; + int64_t head_dim = 0; + float ln_eps = 1e-6f; + bool flash_attn = false; +}; + +struct EncStack { + EncCfg cfg; + std::vector blk; + + // Reads "..*". SigLIP checkpoints prefix their blocks with + // "blk", so callers pass e.g. "vit.blk". + void declare(WeightLoader & L, const char * prefix, int64_t layers, const EncNames & n = EncNames{}); + + ggml_tensor * block(ggml_context * C, const EncBlockW & w, ggml_tensor * x, + int64_t seq, int64_t nv = 1) const; + + ggml_tensor * build(ggml_context * C, ggml_tensor * x, int64_t seq, int64_t nv = 1) const; +}; + +} diff --git a/src/models/vision_common.h b/src/modules/preprocess.h similarity index 60% rename from src/models/vision_common.h rename to src/modules/preprocess.h index 9e33f7b..3e3a2b7 100644 --- a/src/models/vision_common.h +++ b/src/modules/preprocess.h @@ -72,4 +72,49 @@ inline bool preprocess_image_chw(const char * arch, const ImageView & v, int64_t return true; } +// HWC to a [3*ps*ps, grid*grid] patch table in [-1, 1], the GEMM patch-embed +// input GR00T N1.6 uses in place of a conv2d. +inline bool preprocess_image_patches(const char * arch, const ImageView & v, int64_t side, int64_t ps, + std::vector & out) { + if (v.w != (int) side || v.h != (int) side || !v.data) { + std::fprintf(stderr, "vla(%s): image view is %dx%d, expected %lldx%lld\n", + arch, v.w, v.h, (long long) side, (long long) side); + return false; + } + const int64_t grid = side/ps, pd = 3*ps*ps, np = grid*grid; + out.assign((size_t) pd*np, 0.0f); + + auto px = [&](int64_t r, int64_t c, int64_t ch) -> float { + if (v.format == PixelFormat::U8) return ((const uint8_t *) v.data)[(r*side+c)*3+ch]/255.0f; + return ((const float *) v.data)[(r*side+c)*3+ch]; + }; + + for (int64_t row = 0; row < grid; ++row) + for (int64_t col = 0; col < grid; ++col) { + const int64_t t = row*grid+col; + for (int64_t ph = 0; ph < ps; ++ph) + for (int64_t pw = 0; pw < ps; ++pw) + for (int64_t ch = 0; ch < 3; ++ch) + out[t*pd+ph*ps*3+pw*3+ch] = px(row*ps+ph, col*ps+pw, ch)*2.0f-1.0f; + } + return true; +} + +// Pixel shuffle with c-outermost channel order, the inverse layout to +// pixel_shuffle_hf above. GR00T N1.6's connector expects this one. +inline void pixel_shuffle_back(const float * src, int64_t grid, int64_t hidden, int64_t r, float * dst) { + const int64_t g2 = grid/r, c4 = hidden*r*r; + for (int64_t y = 0; y < g2; ++y) + for (int64_t x = 0; x < g2; ++x) { + const int64_t t = y*g2+x; + for (int64_t c = 0; c < hidden; ++c) + for (int64_t i = 0; i < r; ++i) + for (int64_t j = 0; j < r; ++j) { + const int64_t pp = (r*y+i)*grid+(r*x+j); + const int64_t cp = c*r*r+i*r+j; + dst[t*c4+cp] = src[pp*hidden+c]; + } + } +} + } // namespace vla diff --git a/src/modules/prompt.cpp b/src/modules/prompt.cpp new file mode 100644 index 0000000..6352627 --- /dev/null +++ b/src/modules/prompt.cpp @@ -0,0 +1,84 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/prompt.h" + +#include +#include +#include +#include + +namespace vla { + +bool build_prompt(const char * arch, const Inputs & in, int64_t n_img, + int32_t image_token, int64_t max_seq, Prompt & out) { + out = Prompt{}; + + int64_t slots = 0; + for (int j = 0; j < in.n_lang; ++j) + if (in.lang_tokens[j] == image_token) ++slots; + + if (slots == n_img) { + out.ids.assign(in.lang_tokens, in.lang_tokens+in.n_lang); + } else if (slots == 0) { + out.ids.reserve((size_t)(n_img+in.n_lang)); + for (int64_t i = 0; i < n_img; ++i) out.ids.push_back(image_token); + for (int j = 0; j < in.n_lang; ++j) out.ids.push_back(in.lang_tokens[j]); + } else { + std::fprintf(stderr, "vla(%s): lang_tokens has %lld image-token slots but n_img=%lld; expected 0 or %lld\n", + arch, (long long) slots, (long long) n_img, (long long) n_img); + return false; + } + + const int64_t seq = out.len(); + if (seq > max_seq) { + std::fprintf(stderr, "vla(%s): prompt too long (%lld > %lld)\n", arch, (long long) seq, (long long) max_seq); + return false; + } + + out.image_pos.reserve((size_t) n_img); + out.text_pos.reserve((size_t)(seq-n_img)); + for (int64_t p = 0; p < seq; ++p) { + if (out.ids[p] == image_token) out.image_pos.push_back((int32_t) p); + else out.text_pos.push_back((int32_t) p); + } + return true; +} + +bool fetch_embeds(const char * arch, gguf_reader & io, const Prompt & p, + const float * img_emb, int64_t hidden, std::vector & out) { + const int64_t seq = p.len(); + out.assign((size_t) seq*hidden, 0.0f); + if (!io.fetch_rows_f32("token_embd.weight", p.ids, out.data(), hidden)) return false; + + for (size_t k = 0; k < p.image_pos.size(); ++k) + std::memcpy(out.data()+(size_t) p.image_pos[k]*hidden, img_emb+k*hidden, hidden*sizeof(float)); + + (void) arch; + return true; +} + +void init_noise(const Inputs & in, size_t n, std::vector & out) { + out.assign(n, 0.0f); + if (in.noise) { + std::memcpy(out.data(), in.noise, n*sizeof(float)); + return; + } + + std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); + std::normal_distribution nd(0.f, 1.f); + for (auto & v : out) v = nd(rng); +} + +} diff --git a/src/modules/prompt.h b/src/modules/prompt.h new file mode 100644 index 0000000..43f9a16 --- /dev/null +++ b/src/modules/prompt.h @@ -0,0 +1,52 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Token-sequence assembly for the archs whose backbone takes one interleaved +// image/text stream: build the id list, note where the image slots landed, then +// swap the tower's embeddings into those rows. + +#pragma once + +#include "gguf_reader.h" +#include "model.h" + +#include +#include + +namespace vla { + +struct Prompt { + std::vector ids; + std::vector image_pos; + std::vector text_pos; + + int64_t len() const { return (int64_t) ids.size(); } + int64_t n_text() const { return (int64_t) text_pos.size(); } +}; + +// Accepts a caller-supplied stream that already carries exactly n_img image +// placeholders, or one with none, in which case the placeholders are prepended. +// Any other count is a mismatch between the tokenizer and the tower. +bool build_prompt(const char * arch, const Inputs & in, int64_t n_img, + int32_t image_token, int64_t max_seq, Prompt & out); + +// Embedding-table rows for the prompt, with the image rows overwritten by the +// tower output. +bool fetch_embeds(const char * arch, gguf_reader & io, const Prompt & p, + const float * img_emb, int64_t hidden, std::vector & out); + +// The request's noise if it carried any, else a fresh N(0,1) draw. +void init_noise(const Inputs & in, size_t n, std::vector & out); + +} diff --git a/src/modules/qwen3_lm.cpp b/src/modules/qwen3_lm.cpp new file mode 100644 index 0000000..ef66f6e --- /dev/null +++ b/src/modules/qwen3_lm.cpp @@ -0,0 +1,85 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/qwen3_lm.h" + +#include "layers/attn.h" +#include "layers/ffn.h" +#include "layers/norm.h" + +#include + +namespace vla { + +void Qwen3LM::declare(WeightLoader & L, const char * prefix) { + output_norm = L.f32("%s.output_norm.weight", prefix); + blk.resize(cfg.layers); + + for (int64_t i=0; i +#include + +namespace vla { + +struct Qwen3LayerW { + ggml_tensor *attn_norm, *Wq, *Wk, *Wv, *Wo, *q_norm, *k_norm, *ffn_norm, *Wgate, *Wup, *Wdown; +}; + +struct Qwen3Cfg { + int64_t hidden = 2048; + int64_t layers = 16; + int64_t n_q = 16; + int64_t n_kv = 8; + int64_t head_dim = 128; + int64_t inter = 6144; + float rms_eps = 1e-6f; + RopeSpec rope; + bool flash_attn = false; +}; + +struct Qwen3LM { + Qwen3Cfg cfg; + std::vector blk; + ggml_tensor * output_norm = nullptr; + + // Reads ".blk..*" and ".output_norm.weight". + void declare(WeightLoader & L, const char * prefix); + + ggml_tensor * block(ggml_context * C, const Qwen3LayerW & w, ggml_tensor * h, + ggml_tensor * pos, ggml_tensor * mask, int64_t seq) const; + + // Every block, then the output RMSNorm. + ggml_tensor * build(ggml_context * C, ggml_tensor * h, + ggml_tensor * pos, ggml_tensor * mask, int64_t seq) const; +}; + +} diff --git a/src/models/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h similarity index 100% rename from src/models/qwen3vl_vit.h rename to src/modules/qwen3vl_vit.h diff --git a/src/modules/siglip_vit.cpp b/src/modules/siglip_vit.cpp new file mode 100644 index 0000000..5b3dcfc --- /dev/null +++ b/src/modules/siglip_vit.cpp @@ -0,0 +1,51 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "modules/siglip_vit.h" + +#include "layers/norm.h" + +#include + +namespace vla { + +void SigLipTower::declare(WeightLoader & L, const char * prefix, int64_t layers, bool patch_embd_is_gemm) { + patch_w = patch_embd_is_gemm ? L.gemm("%s.patch_embd.weight", prefix) + : L.f32 ("%s.patch_embd.weight", prefix); + patch_b = L.f32("%s.patch_embd.bias", prefix); + pos = L.f32("%s.pos_embd", prefix); + post_ln_w = L.f32("%s.post_ln.weight", prefix); + post_ln_b = L.f32("%s.post_ln.bias", prefix); + + char blk_prefix[192]; + std::snprintf(blk_prefix, sizeof(blk_prefix), "%s.blk", prefix); + enc.declare(L, blk_prefix, layers); +} + +ggml_tensor * SigLipTower::embed_conv(ggml_context * C, ggml_tensor * pixels, int64_t patch, int64_t grid) const { + ggml_tensor * conv = ggml_conv_2d(C, patch_w, pixels, (int)patch, (int)patch, 0, 0, 1, 1); + ggml_tensor * flat = ggml_cont(C, ggml_transpose(C, ggml_reshape_2d(C, conv, grid*grid, enc.cfg.hidden))); + return ggml_add(C, ggml_add(C, flat, patch_b), pos); +} + +ggml_tensor * SigLipTower::embed_patches(ggml_context * C, ggml_tensor * patches) const { + return ggml_add(C, ggml_add(C, ggml_mul_mat(C, patch_w, patches), patch_b), pos); +} + +ggml_tensor * SigLipTower::build(ggml_context * C, ggml_tensor * h, int64_t seq, int64_t nv) const { + h = enc.build(C, h, seq, nv); + return layer_norm(C, h, post_ln_w, post_ln_b, enc.cfg.ln_eps); +} + +} diff --git a/src/modules/siglip_vit.h b/src/modules/siglip_vit.h new file mode 100644 index 0000000..1a88bb9 --- /dev/null +++ b/src/modules/siglip_vit.h @@ -0,0 +1,49 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// SigLIP vision tower: learned position embeddings, post-LN encoder blocks, a +// final LayerNorm, and no CLS token. +// +// The patch embedding has two spellings. GR00T N1.5, pi0, pi0.5 and SmolVLA +// feed a CHW image through a conv2d; N1.6 pre-patchifies on the host and feeds +// a GEMM. Both land on the same [hidden, patches] activation. + +#pragma once + +#include "modules/encoder.h" + +#include "ggml.h" + +#include + +namespace vla { + +struct SigLipTower { + EncStack enc; + ggml_tensor *patch_w = nullptr, *patch_b = nullptr; + ggml_tensor *pos = nullptr; + ggml_tensor *post_ln_w = nullptr, *post_ln_b = nullptr; + + // Reads ".patch_embd.*", ".pos_embd", ".post_ln.*" + // and ".blk..*". + void declare(WeightLoader & L, const char * prefix, int64_t layers, bool patch_embd_is_gemm = false); + + ggml_tensor * embed_conv(ggml_context * C, ggml_tensor * pixels, int64_t patch, int64_t grid) const; + ggml_tensor * embed_patches(ggml_context * C, ggml_tensor * patches) const; + + // Encoder blocks, then the final LayerNorm. + ggml_tensor * build(ggml_context * C, ggml_tensor * h, int64_t seq, int64_t nv = 1) const; +}; + +} diff --git a/src/models/scratch_ctx.h b/src/scratch_ctx.h similarity index 100% rename from src/models/scratch_ctx.h rename to src/scratch_ctx.h diff --git a/tests/bitvla_gemm_check.cu b/tests/bitvla_gemm_check.cu new file mode 100644 index 0000000..74a73b7 --- /dev/null +++ b/tests/bitvla_gemm_check.cu @@ -0,0 +1,238 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file bitvla_gemm_check.cu + * @brief A/B correctness + throughput check for the BitVLA ternary GEMM. + * + * The wide kernel (@c ladder_int8xint2_kernel_m_wide) changes only how work is + * tiled, not the arithmetic: identical int8 x int2 products accumulated in + * int32 and scaled by the same per-row and per-column-group factors. So its + * output must be *bit-identical* to the one-tile-per-CTA kernel, not merely + * close -- an epsilon here would hide a real indexing bug behind bf16 rounding. + * + * Runs every shape the production dispatch supports, at the sequence lengths + * the LM and ViT actually use, plus a ragged M to exercise the row tail. + */ + +#include "../src/kernels/bitvla/bitnet_kernels.h" + +#include +#include +#include +#include +#include +#include + +#define CUDA_OK(expr) \ + do { \ + cudaError_t _e = (expr); \ + if (_e != cudaSuccess) { \ + std::fprintf(stderr, "CUDA error %s at %s:%d\n", cudaGetErrorString(_e), \ + __FILE__, __LINE__); \ + return 2; \ + } \ + } while (0) + +namespace { + +struct Shape { + const char* name; + int N, K, ws_num; +}; + +// Every shape in bitlinear_int8xint2_m's dispatch. +const Shape kShapes[] = { + {"lm.q/o ", 2560, 2560, 1}, + {"lm.k/v ", 640, 2560, 1}, + {"lm.gate_up ", 13824, 2560, 2}, + {"lm.down ", 2560, 6912, 1}, + {"vit.qkvo ", 1152, 1152, 1}, + {"vit.fc1 ", 4304, 1152, 1}, + {"vit.fc2 ", 1152, 4352, 1}, + {"head.qkv ", 3840, 2560, 3}, +}; + +float bf16_to_f32(__nv_bfloat16 h) { + uint16_t u; + std::memcpy(&u, &h, 2); + uint32_t b = ((uint32_t)u) << 16; + float f; + std::memcpy(&f, &b, 4); + return f; +} + +int run_shape(const Shape& sh, int M, int nt, bool& ok) { + const size_t a_sz = (size_t)M * sh.K; + const size_t b_sz = (size_t)sh.N * sh.K / 4; // int2: 4 values per byte + const size_t o_sz = (size_t)M * sh.N; + + std::mt19937 rng(1234u + (unsigned)sh.N + (unsigned)M); + std::vector h_a(a_sz); + std::vector h_b(b_sz); + std::vector h_s(M), h_ws(sh.ws_num); + for (auto& v : h_a) v = (int8_t)((int)(rng() % 255) - 127); + for (auto& v : h_b) v = (int8_t)(rng() % 256); + for (auto& v : h_s) v = 40.0f + (float)(rng() % 100); + for (auto& v : h_ws) v = 0.01f + 0.001f * (float)(rng() % 50); + + int8_t *d_a, *d_b; + __nv_bfloat16 *d_ref, *d_new; + float *d_s, *d_ws; + CUDA_OK(cudaMalloc(&d_a, a_sz)); + CUDA_OK(cudaMalloc(&d_b, b_sz)); + CUDA_OK(cudaMalloc(&d_ref, o_sz * sizeof(__nv_bfloat16))); + CUDA_OK(cudaMalloc(&d_new, o_sz * sizeof(__nv_bfloat16))); + CUDA_OK(cudaMalloc(&d_s, M * sizeof(float))); + CUDA_OK(cudaMalloc(&d_ws, sh.ws_num * sizeof(float))); + CUDA_OK(cudaMemcpy(d_a, h_a.data(), a_sz, cudaMemcpyHostToDevice)); + CUDA_OK(cudaMemcpy(d_b, h_b.data(), b_sz, cudaMemcpyHostToDevice)); + CUDA_OK(cudaMemcpy(d_s, h_s.data(), M * sizeof(float), cudaMemcpyHostToDevice)); + CUDA_OK(cudaMemcpy(d_ws, h_ws.data(), sh.ws_num * sizeof(float), cudaMemcpyHostToDevice)); + CUDA_OK(cudaMemset(d_ref, 0, o_sz * sizeof(__nv_bfloat16))); + CUDA_OK(cudaMemset(d_new, 0, o_sz * sizeof(__nv_bfloat16))); + + // Dispatch by shape, mirroring bitlinear_int8xint2_m. Templated on N/K, so + // this has to be a chain rather than a loop. +#define BOTH(NN, KK, WS) \ + if (sh.N == (NN) && sh.K == (KK)) { \ + launch_ladder_int8xint2_m(d_a, d_b, d_ref, d_s, d_ws, M, 0);\ + if (nt == 1) \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + else if (nt == 2) \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + else \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + } + BOTH(2560, 2560, 1) + BOTH(640, 2560, 1) + BOTH(13824, 2560, 2) + BOTH(2560, 6912, 1) + BOTH(1152, 1152, 1) + BOTH(4304, 1152, 1) + BOTH(1152, 4352, 1) + BOTH(3840, 2560, 3) +#undef BOTH + CUDA_OK(cudaDeviceSynchronize()); + CUDA_OK(cudaGetLastError()); + + std::vector<__nv_bfloat16> h_ref(o_sz), h_new(o_sz); + CUDA_OK(cudaMemcpy(h_ref.data(), d_ref, o_sz * sizeof(__nv_bfloat16), cudaMemcpyDeviceToHost)); + CUDA_OK(cudaMemcpy(h_new.data(), d_new, o_sz * sizeof(__nv_bfloat16), cudaMemcpyDeviceToHost)); + + size_t mismatches = 0; + size_t first = 0; + for (size_t i = 0; i < o_sz; ++i) { + uint16_t a, b; + std::memcpy(&a, &h_ref[i], 2); + std::memcpy(&b, &h_new[i], 2); + if (a != b) { + if (mismatches == 0) first = i; + ++mismatches; + } + } + + // Timing: median of 20 after 5 warmup, so a stray clock excursion does not + // decide the reported speedup. + auto bench = [&](bool wide) -> float { + cudaEvent_t e0, e1; + cudaEventCreate(&e0); + cudaEventCreate(&e1); + std::vector ms; + for (int it = 0; it < 25; ++it) { + cudaEventRecord(e0); +#define TIME_ONE(NN, KK, WS) \ + if (sh.N == (NN) && sh.K == (KK)) { \ + if (!wide) \ + launch_ladder_int8xint2_m(d_a, d_b, d_ref, d_s, d_ws, M, 0);\ + else if (nt == 1) \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + else if (nt == 2) \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + else \ + launch_ladder_int8xint2_m_wide(d_a, d_b, d_new, d_s, d_ws, M, 0); \ + } + TIME_ONE(2560, 2560, 1) + TIME_ONE(640, 2560, 1) + TIME_ONE(13824, 2560, 2) + TIME_ONE(2560, 6912, 1) + TIME_ONE(1152, 1152, 1) + TIME_ONE(4304, 1152, 1) + TIME_ONE(1152, 4352, 1) + TIME_ONE(3840, 2560, 3) +#undef TIME_ONE + cudaEventRecord(e1); + cudaEventSynchronize(e1); + float t = 0; + cudaEventElapsedTime(&t, e0, e1); + if (it >= 5) ms.push_back(t); + } + cudaEventDestroy(e0); + cudaEventDestroy(e1); + std::sort(ms.begin(), ms.end()); + return ms[ms.size() / 2]; + }; + + const float t_ref = bench(false); + const float t_new = bench(true); + const double macs = (double)M * sh.N * sh.K; + + std::printf("%s M=%-4d nt=%d ref %7.3f ms (%5.1f TOPS) wide %7.3f ms (%5.1f TOPS)" + " %4.2fx %s\n", + sh.name, M, nt, t_ref, macs / (t_ref * 1e-3) / 1e12, t_new, + macs / (t_new * 1e-3) / 1e12, t_ref / t_new, + mismatches == 0 ? "bit-identical" + : "MISMATCH"); + if (mismatches != 0) { + std::printf(" %zu/%zu differ, first at %zu: ref %g new %g\n", mismatches, + o_sz, first, bf16_to_f32(h_ref[first]), bf16_to_f32(h_new[first])); + ok = false; + } + + cudaFree(d_a); cudaFree(d_b); cudaFree(d_ref); cudaFree(d_new); + cudaFree(d_s); cudaFree(d_ws); + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + // Default run is the production tiling only; pass "sweep" to compare + // N_TILES = 1/2/4 per shape, which is how the per-shape choice was made. + const bool sweep = (argc > 1 && std::strcmp(argv[1], "sweep") == 0); + bool ok = true; + // 600: the LM sequence length for a LIBERO prompt (512 image markers + 1 + // proprio + prompt + 56 action slots + stop). 256: the ViT, which runs one + // 256-patch view per call rather than batching both. 517: ragged, to + // exercise the M tail (517 = 4*128 + 5). + const int kMs[] = {600, 256, 517}; + const int kSweepTiles[] = {1, 2, 4}; + for (const auto& sh : kShapes) { + for (int m : kMs) { + if (sweep) { + for (int nt : kSweepTiles) { + int rc = run_shape(sh, m, nt, ok); + if (rc) return rc; + } + std::printf("\n"); + } else { + int rc = run_shape(sh, m, bitvla_n_tiles_for(sh.N, sh.K), ok); + if (rc) return rc; + } + } + } + std::printf("\n%s\n", ok ? "PASS: wide kernel is bit-identical on every shape" + : "FAIL: outputs differ"); + return ok ? 0 : 1; +} diff --git a/tests/test_bf16_cuda_ops.cpp b/tests/test_bf16_cuda_ops.cpp index 8b0ed50..b72b29a 100644 --- a/tests/test_bf16_cuda_ops.cpp +++ b/tests/test_bf16_cuda_ops.cpp @@ -30,7 +30,7 @@ #include "ggml-cuda.h" #include "cuda/vla_cuda_ops.h" -#include "models/act_dtype.h" +#include "act_dtype.h" #include #include diff --git a/tests/test_qwen3vl_vit.cpp b/tests/test_qwen3vl_vit.cpp index 1605480..1878249 100644 --- a/tests/test_qwen3vl_vit.cpp +++ b/tests/test_qwen3vl_vit.cpp @@ -14,7 +14,7 @@ // Pins the Qwen3-VL patch geometry. predict_check covers the graph builders. -#include "models/qwen3vl_vit.h" +#include "modules/qwen3vl_vit.h" #include #include diff --git a/tests/test_vision_common.cpp b/tests/test_vision_common.cpp index 5dc5ceb..e0cc70c 100644 --- a/tests/test_vision_common.cpp +++ b/tests/test_vision_common.cpp @@ -15,7 +15,7 @@ // Unit test for the IDEFICS3/SmolVLM pixel-shuffle channel order: a c-outermost // (wrong) arrangement fails the hard-coded expectations below. -#include "models/vision_common.h" +#include "modules/preprocess.h" #undef NDEBUG // keep assert() live even in Release builds #include From b5fb71a9cae034ae58450557cb5715422327b8c4 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 22:13:08 +0700 Subject: [PATCH 07/21] make precision and attention runtime options instead of per-arch env switches --- CMakeLists.txt | 1 + eval/refactor_verify.sh | 105 ++++++++++++-------- src/arch.h | 34 +++++-- src/layers/attn.h | 9 +- src/layers/embed.h | 7 +- src/layers/ffn.h | 5 +- src/layers/linear.h | 6 +- src/layers/norm.h | 5 +- src/layers/rope.h | 18 +--- src/loader.cpp | 4 - src/loader.h | 18 +--- src/model.cpp | 35 ++++--- src/model.h | 4 +- src/models/bitvla.cpp | 6 +- src/models/evo1.cpp | 14 ++- src/models/gr00tn1d5.cpp | 10 +- src/models/gr00tn1d6.cpp | 11 +- src/models/gr00tn1d7.cpp | 10 +- src/models/openvla_oft.cpp | 6 +- src/models/pi0.cpp | 16 ++- src/models/pi05.cpp | 6 +- src/models/smolvla.cpp | 15 +-- src/models/vla_adapter.cpp | 6 +- src/models/vla_jepa.cpp | 6 +- src/modules/action_expert.h | 6 +- src/modules/dit_head.h | 17 +--- src/modules/encoder.h | 9 +- src/modules/preprocess.h | 5 +- src/modules/prompt.h | 13 +-- src/modules/qwen3_lm.h | 5 +- src/modules/qwen3vl_vit.h | 6 +- src/modules/siglip_vit.h | 11 +- src/options.cpp | 193 ++++++++++++++++++++++++++++++++++++ src/options.h | 66 ++++++++++++ src/serving/server.cpp | 19 +++- tests/CMakeLists.txt | 2 + tests/predict_check.cpp | 18 +++- 37 files changed, 497 insertions(+), 230 deletions(-) create mode 100644 src/options.cpp create mode 100644 src/options.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b4992d..89da1ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ vla_exclude_fetched_targets(${llama_SOURCE_DIR}) add_library(vla_core src/model.cpp src/loader.cpp + src/options.cpp src/modules/action_expert.cpp src/modules/dit_head.cpp src/modules/encoder.cpp diff --git a/eval/refactor_verify.sh b/eval/refactor_verify.sh index 4b37e1c..0ec5c33 100755 --- a/eval/refactor_verify.sh +++ b/eval/refactor_verify.sh @@ -1,21 +1,27 @@ #!/usr/bin/env bash # Copyright 2026 VinRobotics - Apache-2.0 # -# Bit-exactness harness for the src/models -> layer/module/model refactor. +# Bit-exactness and latency harness for the src/ layer/module/model refactor. # -# Runs tests/predict_check over every arch's real GGUF with fixed images / -# language / state / noise and writes one action-chunk file per arch. Capture a -# baseline before touching the code, then re-run after each step and diff: +# eval/refactor_verify.sh actions only +# BENCH=20 eval/refactor_verify.sh actions + predict() timing # -# eval/refactor_verify.sh outputs/refactor/base -# ...refactor... -# eval/refactor_verify.sh outputs/refactor/new -# diff -r outputs/refactor/base outputs/refactor/new && echo BIT-EXACT +# Each arch runs twice: at its shipping defaults, and under the alternate +# precision. Both must stay byte-identical across a refactor, and neither may +# regress in latency. # -# ARCHS=... restricts the sweep to a subset (space separated, names below). -# The square input side is probed rather than hardcoded: predict_check defaults -# to 224 and an arch whose tower wants another side returns action_len=0 on a -# mismatch instead of failing, so a wrong side would silently "pass" a diff. +# eval/refactor_verify.sh outputs/refactor/before +# ...change... +# cmake --build build -j"$(nproc)" --target vla_predict_check +# eval/refactor_verify.sh outputs/refactor/after +# diff -r outputs/refactor/before outputs/refactor/after +# +# Never rebuild while a sweep is running: relinking libvla_core.so under it +# makes every remaining arch fail to load. +# +# ARCHS=... restricts the sweep. The square input side is probed rather than +# hardcoded, because a tower fed the wrong side returns action_len=0 instead of +# failing, and a wrong side would silently "pass" a diff. set -euo pipefail @@ -26,29 +32,39 @@ BIN="${BIN:-${REPO_ROOT}/build/tests/vla_predict_check}" HF="${HF:-/mnt/data/hf_data/vrfai}" OUT="${1:-${REPO_ROOT}/outputs/refactor/baseline}" SIDES="${SIDES:-224 256 448 512}" +BENCH="${BENCH:-0}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" -# arch|ckpt|mmproj|n_images|extra env +# arch|ckpt|mmproj|n_images|env|alternate-config CLI flags. +# openvla_oft has no alternate: at f32 its weights need 30 GB. MODELS=( - "smolvla|${HF}/smolvla-libero-gguf/smolvla-libero.gguf|${HF}/backup/mmproj-smolvla-libero.gguf|2|" - "pi0|${HF}/pi0-libero-finetuned-v044-gguf/pi0-libero-finetuned-v044.gguf|${HF}/backup/mmproj-pi0-libero-finetuned-v044.gguf|2|" - "pi05|${HF}/pi05-libero-gguf/pi05-libero.gguf|${HF}/backup/mmproj-pi05-libero.gguf|2|" - "evo1|${HF}/evo1-libero-gguf/evo1-libero.gguf||2|" - "gr00t_n1_5|${HF}/gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf||2|" - "gr00t_n1_6|${HF}/gr00tn1d6-libero-gguf/gr00tn1d6-libero.gguf||2|" - "gr00t_n1_7|${HF}/gr00tn1d7-libero-gguf/libero_object/gr00tn1d7-libero-object.gguf||2|" - "bitvla|${HF}/bitvla-libero-gguf/libero_object/bitvla-libero-object.gguf||2|" - "vla_adapter|${HF}/vla-adapter-libero-object-gguf/libero_object/vla-adapter-libero-object.gguf||2|" - "openvla_oft|${HF}/openvla-oft-libero-gguf/openvla-oft-libero.gguf||2|" - "vla_jepa|${HF}/vla-jepa-libero/vla-jepa.gguf||2|VLA_EXTRA_TOKEN=151697 VLA_EXTRA_COUNT=32" + "smolvla|${HF}/smolvla-libero-gguf/smolvla-libero.gguf|${HF}/backup/mmproj-smolvla-libero.gguf|2||--flash-attn --mm-prec default" + "pi0|${HF}/pi0-libero-finetuned-v044-gguf/pi0-libero-finetuned-v044.gguf|${HF}/backup/mmproj-pi0-libero-finetuned-v044.gguf|2||--act-dtype bf16 --flash-attn" + "pi05|${HF}/pi05-libero-gguf/pi05-libero.gguf|${HF}/backup/mmproj-pi05-libero.gguf|2||--weight-dtype f32" + "evo1|${HF}/evo1-libero-gguf/evo1-libero.gguf||2||--act-dtype bf16 --flash-attn" + "gr00t_n1_5|${HF}/gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf||2||--weight-dtype f32" + "gr00t_n1_6|${HF}/gr00tn1d6-libero-gguf/gr00tn1d6-libero.gguf||2||--weight-dtype f32" + "gr00t_n1_7|${HF}/gr00tn1d7-libero-gguf/libero_object/gr00tn1d7-libero-object.gguf||2||--weight-dtype f32" + "bitvla|${HF}/bitvla-libero-gguf/libero_object/bitvla-libero-object.gguf||2||--weight-dtype bf16" + "vla_adapter|${HF}/vla-adapter-libero-object-gguf/libero_object/vla-adapter-libero-object.gguf||2||--weight-dtype f32" + "openvla_oft|${HF}/openvla-oft-libero-gguf/openvla-oft-libero.gguf||2||" + "vla_jepa|${HF}/vla-jepa-libero/vla-jepa.gguf||2|VLA_EXTRA_TOKEN=151697 VLA_EXTRA_COUNT=32|--weight-dtype f32" ) [[ -x "${BIN}" ]] || { echo "ERROR: missing ${BIN} (cmake -DVLA_BUILD_TESTS=ON)" >&2; exit 1; } mkdir -p "${OUT}" +run_one() { + local arch="$1" ckpt="$2" mmproj="$3" nimg="$4" env_str="$5" tag="$6" side="$7" cli="$8" + # shellcheck disable=SC2086 + env ${env_str} VLA_IMG_SIZE="${side}" VLA_BENCH_ITERS="${BENCH}" \ + "${BIN}" "${ckpt}" "${mmproj}" "${nimg}" ${cli} \ + > "${OUT}/${arch}${tag}.actions.txt" 2> "${OUT}/${arch}${tag}.log" +} + fail=0 for row in "${MODELS[@]}"; do - IFS='|' read -r arch ckpt mmproj nimg extra <<< "${row}" + IFS='|' read -r arch ckpt mmproj nimg always fastest <<< "${row}" if [[ -n "${ARCHS:-}" && " ${ARCHS} " != *" ${arch} "* ]]; then continue @@ -58,26 +74,37 @@ for row in "${MODELS[@]}"; do continue fi - ok=0 - for side in ${SIDES}; do - # shellcheck disable=SC2086 - if env ${extra} VLA_IMG_SIZE="${side}" "${BIN}" "${ckpt}" "${mmproj}" "${nimg}" \ - > "${OUT}/${arch}.actions.txt" 2> "${OUT}/${arch}.log"; then - if ! grep -q '^action_len=0$' "${OUT}/${arch}.actions.txt"; then - echo "[ok ] ${arch} side=${side} $(head -1 "${OUT}/${arch}.actions.txt")" - echo "${side}" > "${OUT}/${arch}.side" - ok=1 - break - fi + side="" + for s in ${SIDES}; do + if run_one "${arch}" "${ckpt}" "${mmproj}" "${nimg}" "${always}" "" "${s}" "" \ + && ! grep -q '^action_len=0$' "${OUT}/${arch}.actions.txt"; then + side="${s}" + echo "${s}" > "${OUT}/${arch}.side" + break fi done - - if [[ "${ok}" -eq 0 ]]; then + if [[ -z "${side}" ]]; then echo "[FAIL] ${arch}: no input side in '${SIDES}' produced a chunk; see ${OUT}/${arch}.log" >&2 fail=1 + continue + fi + + line="[ok ] ${arch} side=${side}" + [[ "${BENCH}" -gt 0 ]] && line+=" default=$(grep -oP 'min=\K[0-9.]+' "${OUT}/${arch}.log" | head -1)ms" + + if [[ -n "${fastest}" ]]; then + if run_one "${arch}" "${ckpt}" "${mmproj}" "${nimg}" "${always}" ".alt" "${side}" "${fastest}" \ + && ! grep -q '^action_len=0$' "${OUT}/${arch}.alt.actions.txt"; then + line+=" alt=ok" + [[ "${BENCH}" -gt 0 ]] && line+=" $(grep -oP 'min=\K[0-9.]+' "${OUT}/${arch}.alt.log" | head -1)ms" + else + echo "[FAIL] ${arch}: alternate config produced no chunk; see ${OUT}/${arch}.fast.log" >&2 + fail=1 + fi fi + echo "${line}" done echo -echo "actions written to ${OUT}" +echo "written to ${OUT}" exit "${fail}" diff --git a/src/arch.h b/src/arch.h index e66fa33..4a10b34 100644 --- a/src/arch.h +++ b/src/arch.h @@ -26,6 +26,7 @@ #pragma once #include "model.h" +#include "options.h" #include #include @@ -109,7 +110,8 @@ class ModelArchBase { */ std::unique_ptr smolvla_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a pi0 model from its mmproj and checkpoint GGUFs. @@ -117,7 +119,8 @@ std::unique_ptr smolvla_create(const std::string& mmproj_path, */ std::unique_ptr pi0_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a pi0.5 model from its mmproj and checkpoint GGUFs. @@ -125,7 +128,8 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, */ std::unique_ptr pi05_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build an Evo-1 model. Vision is baked into @p ckpt_path; pass @@ -134,7 +138,8 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, */ std::unique_ptr evo1_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a GR00T N1.5 model. Vision is baked into @p ckpt_path. @@ -142,7 +147,8 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, */ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a GR00T N1.6 model. Vision is baked into @p ckpt_path. @@ -150,7 +156,8 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, */ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a GR00T N1.7 model. Vision is baked into @p ckpt_path. @@ -158,7 +165,8 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, */ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a BitVLA model. Vision is baked into @p ckpt_path. @@ -166,7 +174,8 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, */ std::unique_ptr bitvla_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a VLA-Adapter model. Vision is baked into @p ckpt_path. @@ -174,7 +183,8 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, */ std::unique_ptr vla_adapter_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a OpenVLA-OFT model. Vision is baked into @p ckpt_path. @@ -182,7 +192,8 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path */ std::unique_ptr openvla_oft_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Build a VLA-JEPA model. Vision is baked into @p ckpt_path. @@ -190,7 +201,8 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path */ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path); + const std::string& config_path, + const Options& opts); /** * @brief Inspect a GGUF and identify the architecture tag. diff --git a/src/layers/attn.h b/src/layers/attn.h index 70c73a8..bef6af2 100644 --- a/src/layers/attn.h +++ b/src/layers/attn.h @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Scaled dot-product attention. `nv` is the batch (view) count; ggml pads every -// shape to four dimensions, so nv == 1 emits the same nodes as the 2d/3d -// spelling it replaces. + +// nv == 1 emits the same nodes as the 2d/3d spelling, since ggml pads every +// shape to four dimensions. #pragma once @@ -24,7 +24,6 @@ namespace vla { -// [heads*hd, T, nv] -> [hd, T, heads, nv]. inline ggml_tensor * to_heads(ggml_context * C, ggml_tensor * p, int64_t hd, int64_t heads, int64_t T, int64_t nv = 1) { return ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, p, hd, heads, T, nv), 0, 2, 1, 3)); @@ -36,8 +35,6 @@ inline ggml_tensor * to_heads_v(ggml_context * C, ggml_tensor * p, int64_t hd, i return ggml_cont(C, ggml_permute(C, ggml_reshape_4d(C, p, hd, heads, T, nv), 1, 2, 0, 3)); } -// Scores stay F32 whatever the activation dtype: softmax over a BF16 reduction -// loses too much. inline ggml_tensor * attention(ggml_context * C, ggml_tensor * Q, ggml_tensor * K, ggml_tensor * V, ggml_tensor * mask, float scale, int64_t dim, int64_t T, int64_t nv = 1) { ggml_tensor * kq = ggml_mul_mat(C, K, Q); diff --git a/src/layers/embed.h b/src/layers/embed.h index 030c0d1..4439020 100644 --- a/src/layers/embed.h +++ b/src/layers/embed.h @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Host-side input tables, uploaded as graph inputs rather than built as nodes. // The sin/cos order differs per family and each matches its reference; // tests/test_dit_common.cpp pins all three. @@ -43,7 +42,7 @@ inline void timesteps_proj(int64_t bucket, std::vector & out) { } } -// Broadcast across the horizon. sin first, then cos. +// sin first, then cos. inline void action_sinusoid(int64_t bucket, int64_t dim, int64_t T, std::vector & out) { const int64_t half = dim/2; const float step = std::log(10000.0f)/(float)half; @@ -58,8 +57,7 @@ inline void action_sinusoid(int64_t bucket, int64_t dim, int64_t T, std::vector< } } -// Log-spaced periods rather than frequencies, the openpi convention shared by -// pi0, pi0.5 and SmolVLA. +// Log-spaced periods rather than frequencies, the openpi convention. inline std::vector sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { const int64_t half = dim/2; @@ -74,7 +72,6 @@ inline std::vector sinusoidal_time_emb(double t, int64_t dim, double min_ return out; } -// Additive causal mask, -inf above the diagonal. inline void build_causal_mask(int64_t seq, std::vector & out) { const float NEG = -std::numeric_limits::infinity(); diff --git a/src/layers/ffn.h b/src/layers/ffn.h index 9b98d5f..f514d93 100644 --- a/src/layers/ffn.h +++ b/src/layers/ffn.h @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Feed-forward blocks. #pragma once @@ -22,19 +21,17 @@ namespace vla { -// tanh-approximate GELU. inline ggml_tensor * ffn_gelu(ggml_context * C, ggml_tensor * W1, ggml_tensor * b1, ggml_tensor * W2, ggml_tensor * b2, ggml_tensor * x) { return linear(C, W2, b2, ggml_gelu(C, linear(C, W1, b1, x))); } -// Exact erf GELU, what DINOv2 and SigLIP-so400m were trained with. +// DINOv2 and SigLIP-so400m were trained with the exact erf form. inline ggml_tensor * ffn_gelu_erf(ggml_context * C, ggml_tensor * W1, ggml_tensor * b1, ggml_tensor * W2, ggml_tensor * b2, ggml_tensor * x) { return linear(C, W2, b2, ggml_gelu_erf(C, linear(C, W1, b1, x))); } -// down(silu(gate(x)) * up(x)). inline ggml_tensor * ffn_swiglu(ggml_context * C, ggml_tensor * Wg, ggml_tensor * bg, ggml_tensor * Wu, ggml_tensor * bu, ggml_tensor * Wd, ggml_tensor * bd, ggml_tensor * x) { diff --git a/src/layers/linear.h b/src/layers/linear.h index 0f8daa7..dd3a7a7 100644 --- a/src/layers/linear.h +++ b/src/layers/linear.h @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Projections. A null bias emits no add node. #pragma once @@ -27,8 +26,7 @@ inline ggml_tensor * linear(ggml_context * C, ggml_tensor * W, ggml_tensor * b, return b ? ggml_add(C, y, b) : y; } -// One row of a stacked [out, in, n_embodiment] weight: the GR00T action expert -// keeps a per-embodiment copy of every projection in one tensor. +// One row of a stacked [out, in, n_embodiment] weight. inline ggml_tensor * cat_linear(ggml_context * C, ggml_tensor * W3d, ggml_tensor * b2d, int64_t id, ggml_tensor * x) { const int64_t out = W3d->ne[0]; const int64_t in = W3d->ne[1]; @@ -38,7 +36,7 @@ inline ggml_tensor * cat_linear(ggml_context * C, ggml_tensor * W3d, ggml_tensor return ggml_add(C, y, ggml_view_1d(C, b2d, out, (size_t)id*b2d->nb[1])); } -// Slice block `blk` out of a fused [nblk*E, T] projection, laid out as heads. +// Block `blk` of a fused [nblk*E, T] projection, laid out as heads. inline ggml_tensor * head_view(ggml_context * C, ggml_tensor * proj, int64_t hd, int64_t heads, int64_t T, int64_t E, int nblk, int blk) { const size_t es = ggml_element_size(proj); diff --git a/src/layers/norm.h b/src/layers/norm.h index 93cd3e5..a10acf7 100644 --- a/src/layers/norm.h +++ b/src/layers/norm.h @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Normalisation layers. #pragma once @@ -29,14 +28,12 @@ inline ggml_tensor * layer_norm(ggml_context * C, ggml_tensor * x, ggml_tensor * return ggml_add(C, ggml_mul(C, ggml_norm(C, x, eps), w), b); } -// w == nullptr leaves the norm unscaled. inline ggml_tensor * rms_norm(ggml_context * C, ggml_tensor * x, ggml_tensor * w, float eps) { ggml_tensor * n = ggml_rms_norm(C, x, eps); return w ? ggml_mul(C, n, w) : n; } -// The conditioning vector is (scale, shift) in that order; the final projection -// layer of each DiT head uses (shift, scale) instead. +// cond is (scale, shift) here; the DiT final projection uses (shift, scale). inline ggml_tensor * adaln(ggml_context * C, ggml_tensor * x, ggml_tensor * temb, ggml_tensor * lw, ggml_tensor * lb, int64_t dim, float eps) { ggml_tensor * cond = linear(C, lw, lb, ggml_silu(C, temb)); diff --git a/src/layers/rope.h b/src/layers/rope.h index 8874bbd..ca06156 100644 --- a/src/layers/rope.h +++ b/src/layers/rope.h @@ -12,16 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Rotary embeddings. Three conventions live in the tree and they are not -// interchangeable; the pairing is pinned by tests/test_rope_conventions.cpp. -// -// RopeSpec ggml rope, NEOX or IMROPE. The Qwen3 backbones. -// rope_2d precomputed tables, half-split rotation. The Qwen3-VL tower, -// whose 2d grid positions ggml_rope has no spelling for. -// rope_pairwise precomputed tables, adjacent-pair rotation. VLA-Adapter's -// action head pairs a half-split frequency table with an -// interleaved rotation; the reference does the same, so the -// mismatch is deliberate. +// Three conventions, not interchangeable; pinned by +// tests/test_rope_conventions.cpp. rope_pairwise deliberately pairs a +// half-split frequency table with an interleaved rotation, as the VLA-Adapter +// reference does. #pragma once @@ -31,8 +25,6 @@ namespace vla { -// Which ggml rope call a backbone wants, and with what parameters. `sections` -// is read only when type is GGML_ROPE_TYPE_IMROPE. struct RopeSpec { int type = GGML_ROPE_TYPE_NEOX; int n_dims = 0; @@ -55,7 +47,6 @@ inline ggml_tensor * rope(ggml_context * C, const RopeSpec & r, ggml_tensor * x, r.freq_base, r.freq_scale, r.ext_factor, r.attn_factor, r.beta_fast, r.beta_slow); } -// Half-split rotation against precomputed tables: (x1, x2) -> (-x2, x1). inline ggml_tensor * rope_2d(ggml_context * C, ggml_tensor * x, ggml_tensor * cos_t, ggml_tensor * sin_t) { const int64_t hd = x->ne[0]; const int64_t S = x->ne[1]; @@ -68,7 +59,6 @@ inline ggml_tensor * rope_2d(ggml_context * C, ggml_tensor * x, ggml_tensor * co return ggml_add(C, ggml_mul(C, x, cos_t), ggml_mul(C, rot, sin_t)); } -// Adjacent-pair rotation: (even, odd) -> (-odd, even). inline ggml_tensor * rope_pairwise_rot(ggml_context * C, ggml_tensor * x, int64_t HD) { const int64_t L = x->ne[1]; const int64_t H = x->ne[2]; diff --git a/src/loader.cpp b/src/loader.cpp index ab7e0ae..3bb2b4a 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -22,8 +22,6 @@ namespace vla { namespace { -// GGUF tensor names in this tree top out well under this; the truncation guard -// below turns an overflow into a load failure rather than a silent wrong name. constexpr size_t NAME_CAP = 256; } @@ -59,8 +57,6 @@ ggml_tensor * WeightLoader::declare(ggml_type want, bool required, bool gemma_no return t; } -// The five entry points differ only in the resident type and whether a miss is -// fatal, so each is a one-line forward into declare(). #define VLA_DECLARE_FN(fn, type, required, gemma) \ ggml_tensor * WeightLoader::fn(const char * fmt, ...) { \ va_list ap; \ diff --git a/src/loader.h b/src/loader.h index 3f52bcd..f7264cf 100644 --- a/src/loader.h +++ b/src/loader.h @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Weight declaration and upload. A miss is recorded on the loader and surfaces -// once, at ok(), so a module's declare() can stay a flat list of names. -// // gemm() lands in the model's matmul type, unless the GGUF holds the tensor // quantized, in which case it stays packed and ggml dequantizes at compute. -// f32() is for norms, biases and embedding tables. +// A miss is recorded and surfaces once, at ok(). #pragma once @@ -40,29 +37,24 @@ class WeightLoader { WeightLoader(const WeightLoader &) = delete; WeightLoader & operator=(const WeightLoader &) = delete; - // printf-formatted so a per-block prefix needs no scratch buffer at the - // call site. A miss returns nullptr and fails ok(). ggml_tensor * gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); ggml_tensor * f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); - // A miss is not an error; the caller branches on nullptr. + // A miss is not an error. ggml_tensor * opt_gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); ggml_tensor * opt_f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); - // Gemma norms are centred on zero and add 1 at use; folding the +1 in at - // load keeps it off the graph and needs unpacked floats. + // Gemma norms are centred on zero and add 1 at use. ggml_tensor * f32_gemma_norm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); - // One resident tensor holding several GGUF tensors concatenated along the - // outer dimension, so a split projection can be issued as a single GEMM. - // `out_name` is synthetic and need not exist in the file. + // Several GGUF tensors concatenated into one resident tensor; out_name is + // synthetic and need not exist in the file. ggml_tensor * fuse_gemm(const char * out_name, const std::vector & srcs); ggml_tensor * fuse_f32 (const char * out_name, const std::vector & srcs); ggml_type gemm_type() const { return gemm_; } bool ok() const { return ok_; } - // One backend buffer for everything declared so far, then fills it. bool upload(ggml_backend_t backend, ggml_backend_buffer_t * out_buf); private: diff --git a/src/model.cpp b/src/model.cpp index 3b09ceb..e17a2f6 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -149,7 +149,7 @@ bool detect_arch_from_ckpt(const std::string& ckpt_path, Arch* out) { } Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path) { + const std::string& config_path, const Options& opts) { Arch arch; if (!detect_arch_from_ckpt(ckpt_path, &arch)) { std::fprintf(stderr, @@ -160,50 +160,61 @@ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, } std::unique_ptr impl; + { + std::string err; + if (!Options::reject_retired_env(err)) { + std::fprintf(stderr, "vla: %s\n", err.c_str()); + return nullptr; + } + } + + set_flash_attn(opts.flash_attn.value_or(false)); + set_mm_prec_f32(opts.mm_prec_f32.value_or(true)); + switch (arch) { case Arch::SMOLVLA: std::printf("vla: arch = smolvla\n"); - impl = smolvla_create(mmproj_path, ckpt_path, config_path); + impl = smolvla_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::PI0: std::printf("vla: arch = pi0\n"); - impl = pi0_create(mmproj_path, ckpt_path, config_path); + impl = pi0_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::PI05: std::printf("vla: arch = pi05\n"); - impl = pi05_create(mmproj_path, ckpt_path, config_path); + impl = pi05_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::EVO1: std::printf("vla: arch = evo1\n"); - impl = evo1_create(mmproj_path, ckpt_path, config_path); + impl = evo1_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::GR00T_N1_5: std::printf("vla: arch = gr00t_n1_5\n"); - impl = gr00t_n1_5_create(mmproj_path, ckpt_path, config_path); + impl = gr00t_n1_5_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::GR00T_N1_6: std::printf("vla: arch = gr00t_n1_6\n"); - impl = gr00t_n1_6_create(mmproj_path, ckpt_path, config_path); + impl = gr00t_n1_6_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::GR00T_N1_7: std::printf("vla: arch = gr00t_n1_7\n"); - impl = gr00t_n1_7_create(mmproj_path, ckpt_path, config_path); + impl = gr00t_n1_7_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::BITVLA: std::printf("vla: arch = bitvla\n"); - impl = bitvla_create(mmproj_path, ckpt_path, config_path); + impl = bitvla_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::VLA_ADAPTER: std::printf("vla: arch = vla_adapter\n"); - impl = vla_adapter_create(mmproj_path, ckpt_path, config_path); + impl = vla_adapter_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::OPENVLA_OFT: std::printf("vla: arch = openvla_oft\n"); - impl = openvla_oft_create(mmproj_path, ckpt_path, config_path); + impl = openvla_oft_create(mmproj_path, ckpt_path, config_path, opts); break; case Arch::VLA_JEPA: std::printf("vla: arch = vla_jepa\n"); - impl = vla_jepa_create(mmproj_path, ckpt_path, config_path); + impl = vla_jepa_create(mmproj_path, ckpt_path, config_path, opts); break; } if (!impl) return nullptr; diff --git a/src/model.h b/src/model.h index 6265f34..bf4372f 100644 --- a/src/model.h +++ b/src/model.h @@ -26,6 +26,8 @@ #pragma once +#include "options.h" + #include #include #include @@ -164,7 +166,7 @@ struct Inputs { * @return Owning handle. Free with @ref model_free. */ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path = ""); + const std::string& config_path = "", const Options& opts = Options{}); /** * @brief Release a model handle returned by @ref model_load. diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index c3f6a68..05347c3 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -531,13 +532,14 @@ BitvlaModelArch::~BitvlaModelArch() { std::unique_ptr bitvla_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(bitvla): note - mmproj '%s' is ignored (the BitSigLIP-L vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_BITVLA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_F32); gguf_reader g("bitvla"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index af35921..26bf10a 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -199,10 +200,6 @@ bool preprocess_image_chw(const ImageView & v, int64_t side, std::vector // (n=100, same binary). That is inside sampling noise at ~1.6 SE, but the drop // concentrated in the two tasks the control aced, so the default stays on the // accuracy-preserving path and the speedup is opt-in. -inline bool evo1_vit_fa_enabled() { - static const bool enabled = vla::env_flag("VLA_EVO1_FA"); - return enabled; -} ggml_tensor * evo1_flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, float scale, int64_t hidden, int64_t N) { @@ -232,7 +229,7 @@ ggml_tensor * build_internvit_layer(ggml_context * C, const Evo1ModelArch & m, c ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, n_heads, N), 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, n_heads, N), 0, 2, 1, 3)); ggml_tensor * att; - if (evo1_vit_fa_enabled()) { + if (vla::flash_attn_enabled()) { ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, n_heads, N), 0, 2, 1, 3)); att = evo1_flash_attn(C, Q, K, V, scale, H, N); } else { @@ -348,14 +345,15 @@ Evo1ModelArch::~Evo1ModelArch() { std::unique_ptr evo1_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(evo1): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_EVO1_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; @@ -376,7 +374,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. - if (vla::env_flag("VLA_EVO1_BF16_ACT")) { + if (opts.act_dtype.value_or(GGML_TYPE_F32) == GGML_TYPE_BF16) { if (b.is_cuda && m->matmul_type == GGML_TYPE_BF16) { m->act_type = GGML_TYPE_BF16; cuda_register_bf16_ops(); // installs the in-tree BF16 CUDA kernels diff --git a/src/models/gr00tn1d5.cpp b/src/models/gr00tn1d5.cpp index 21a50ae..2d6292d 100644 --- a/src/models/gr00tn1d5.cpp +++ b/src/models/gr00tn1d5.cpp @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// NVIDIA Isaac GR00T N1.5: SigLIP tower -> Qwen3 backbone -> VLSA encoder -> -// DiT action head under a flow-matching solver. - #include "arch.h" +#include "options.h" #include "backend.h" #include "env_flag.h" #include "gguf_reader.h" @@ -199,12 +197,13 @@ Gr00tN1d5ModelArch::~Gr00tN1d5ModelArch() { std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(gr00tn1d5): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) return nullptr; @@ -322,7 +321,6 @@ std::vector Gr00tN1d5ModelArch::predict(const Inputs& in) { std::vector x_init; init_noise(in, (size_t) AH*AD, x_init); - // LM + VLSA + DiT graph depends only on the padded length and step count. const MainKey mkey{ SEQ, num_steps }; const bool built = main_graph.ensure(backend, mkey, (size_t) 128*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index 980cbe5..ebbac5f 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// NVIDIA Isaac GR00T N1.6: SigLIP2 tower with a GEMM patch embed and a -// pixel-shuffled MLP connector, a Qwen3 backbone, and an AlternateVL DiT head -// that cross-attends text and image tokens on alternating blocks. - #include "arch.h" +#include "options.h" #include "backend.h" #include "env_flag.h" #include "gguf_reader.h" @@ -233,12 +230,13 @@ Gr00tN1d6ModelArch::~Gr00tN1d6ModelArch() { std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(gr00tn1d6): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) return nullptr; @@ -396,7 +394,6 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { std::vector x_init; init_noise(in, (size_t) AH*AD, x_init); - // LM + DiT graph depends only on the sequence split and step count. const MainKey mkey{ SEQ, n_img, SEQ_TXT, num_steps }; const bool built = main_graph.ensure(backend, mkey, (size_t) 256*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index fbd6b3b..8862385 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -136,7 +137,7 @@ ggml_tensor * build_vlsa_layer(ggml_context * C, const VlsaLayerW & w, ggml_tens ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, seq), 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, seq), 0, 2, 1, 3)); ggml_tensor * att; - if (fa_enabled()) { + if (vla::flash_attn_enabled()) { ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 0, 2, 1, 3)); att = flash_attn(C, Q, K, V, nullptr, scale); } else { @@ -170,7 +171,7 @@ ggml_tensor * build_qwen3_layer(ggml_context * C, const Gr00tN1d7ModelArch & m, ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); ggml_tensor * att; - if (fa_enabled()) { + if (vla::flash_attn_enabled()) { ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 0, 2, 1, 3)); att = flash_attn(C, Q, K, V, ggml_cast(C, mask, GGML_TYPE_F16), scale); } else { @@ -306,13 +307,14 @@ Gr00tN1d7ModelArch::~Gr00tN1d7ModelArch() { std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(gr00tn1d7): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_GR00T_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("gr00tn1d7"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index 19c6b08..cab6e97 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "modules/preprocess.h" #include "modules/dual_tower.h" @@ -131,11 +132,12 @@ struct OpenVlaOftModelArch : public ModelArchBase { std::unique_ptr openvla_oft_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(openvla_oft): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = vla::env_flag("VLA_OPENVLA_OFT_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("openvla_oft"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 027b57a..4ec1ba7 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -144,10 +145,6 @@ namespace { // measured, so it stays opt-in on an unquantified risk rather than a measured // cost. (The evo1 SR drop this used to cite did not reproduce.) // VLA_PI0_BF16_ACT is the better lever here: 9.1%, and its SR was measured. -static inline bool pi0_fa_enabled() { - static const bool enabled = vla::env_flag("VLA_PI0_FA"); - return enabled; -} ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps, @@ -160,7 +157,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_ ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, head_dim, heads, seq), 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, head_dim, heads, seq), 0, 2, 1, 3)); ggml_tensor * att; - if (pi0_fa_enabled()) { + if (vla::flash_attn_enabled()) { // Avoids materialising the per-head score matrix; K/V stay F32 so the // numerics track the explicit path below. ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, head_dim, heads, seq), 0, 2, 1, 3)); @@ -227,7 +224,7 @@ ggml_tensor * build_gemma_layer( ggml_tensor * Q = ggml_cont(ctx, ggml_permute(ctx, q_rope, 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(ctx, ggml_permute(ctx, K_full, 0, 2, 1, 3)); ggml_tensor * att_pre; - if (pi0_fa_enabled()) { + if (vla::flash_attn_enabled()) { ggml_tensor * V = ggml_cont(ctx, ggml_permute(ctx, V_full, 0, 2, 1, 3)); // ggml_flash_attn_ext asserts an F16 mask. The mask holds only 0 and // -inf, both exactly representable in F16, so the cast is lossless. @@ -350,7 +347,8 @@ Pi0ModelArch::~Pi0ModelArch() { std::unique_ptr pi0_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path) { + const std::string& config_path, + const Options& opts) { (void) config_path; if (!ends_with(ckpt_path, ".gguf")) { @@ -363,7 +361,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = vla::env_flag("VLA_PI0_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; @@ -390,7 +388,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. - if (vla::env_flag("VLA_PI0_BF16_ACT")) { + if (opts.act_dtype.value_or(GGML_TYPE_F32) == GGML_TYPE_BF16) { if (b.is_cuda && m->matmul_type == GGML_TYPE_BF16) { m->act_type = GGML_TYPE_BF16; cuda_register_bf16_ops(); // installs the in-tree BF16 CUDA kernels diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index ccc9114..54acd0e 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -386,7 +387,8 @@ Pi05ModelArch::~Pi05ModelArch() { std::unique_ptr pi05_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path) { + const std::string& config_path, + const Options& opts) { (void) config_path; if (!ends_with(ckpt_path, ".gguf")) { @@ -398,7 +400,7 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = vla::env_flag("VLA_PI05_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); if (!m->io.open(ckpt_path)) return nullptr; gguf_reader & g = m->io; diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index c06ce17..e86e073 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -16,6 +16,7 @@ // distilled action-expert weights and force num_steps = 1 at the denoise loops. #include "arch.h" +#include "options.h" #include "model.h" #include "modules/preprocess.h" #include "scratch_ctx.h" @@ -363,10 +364,6 @@ namespace { // reinterpreting it as F16), and that measured 92/100 on libero_object against // 96/100 for explicit attention. evo1 showed the same ~4-5 pp drop, so the // default stays on the accuracy-preserving path. -static inline bool siglip_fa_enabled() { - static const bool enabled = vla::env_flag("VLA_SMOLVLA_FA"); - return enabled; -} // One pre-norm SigLIP encoder block (SmolVLM2 tower), same graph as the other // in-tree models. Bidirectional attention, F32 score accumulation, tanh GELU. @@ -380,7 +377,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_ ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, head_dim, heads, seq), 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, head_dim, heads, seq), 0, 2, 1, 3)); ggml_tensor * att; - if (siglip_fa_enabled()) { + if (vla::flash_attn_enabled()) { // The tower runs 1024 tokens (512/16 grid) over 12 layers, so the // explicit path below materialises a 1024x1024 score matrix per head — // written by the matmul, read and rewritten by the softmax, then read @@ -724,10 +721,7 @@ ggml_tensor * rope_q_or_k(ggml_context * ctx, ggml_tensor * x, 32.f, 1.f); } -static inline bool tower_mm_f32_prec() { - const char * e = std::getenv("VLA_MM_PREC"); - return !(e && std::strcmp(e, "default") == 0); -} +static inline bool tower_mm_f32_prec() { return vla::mm_prec_f32_enabled(); } static inline ggml_tensor * mm_w(ggml_context * ctx, ggml_tensor * w, ggml_tensor * x) { ggml_tensor * r = ggml_mul_mat(ctx, w, x); if (tower_mm_f32_prec()) ggml_mul_mat_set_prec(r, GGML_PREC_F32); @@ -1897,7 +1891,8 @@ std::vector SmolVLAModelArch::predict(const Inputs& in) { std::unique_ptr smolvla_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& config_path) { + const std::string& config_path, + const Options& opts) { SmolVLAModelArch* raw = smolvla_load_impl(mmproj_path, ckpt_path, config_path); if (!raw) return nullptr; return std::unique_ptr(raw); diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 82e74d0..dbadc8e 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "modules/preprocess.h" #include "modules/dual_tower.h" @@ -155,11 +156,12 @@ static ggml_tensor* hrope(ggml_context*C, ggml_tensor*x, ggml_tensor*cs, ggml_te std::unique_ptr vla_adapter_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(vla_adapter): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = vla::env_flag("VLA_ADAPTER_F32_WEIGHTS") ? GGML_TYPE_F32 : GGML_TYPE_BF16; + m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("vla_adapter"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 9bee501..188090c 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "options.h" #include "model.h" #include "ggml.h" @@ -233,13 +234,14 @@ VlaJepaModelArch::~VlaJepaModelArch() { std::unique_ptr vla_jepa_create(const std::string& mmproj_path, const std::string& ckpt_path, - const std::string& ) { + const std::string&, + const Options& opts) { if (!mmproj_path.empty()) std::printf("vla(vla_jepa): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = vla::env_flag("VLA_JEPA_BF16_WEIGHTS") ? GGML_TYPE_BF16 : GGML_TYPE_F32; + m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("vla_jepa"); if (!g.open(ckpt_path)) return nullptr; diff --git a/src/modules/action_expert.h b/src/modules/action_expert.h index 62c558f..0804a74 100644 --- a/src/modules/action_expert.h +++ b/src/modules/action_expert.h @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// GR00T action expert: the per-embodiment MLPs that lift state and noisy -// actions into the DiT token stream and project the DiT output back to action -// space. Every projection is a cat_linear row selected by embodiment_id. +// Every projection is a cat_linear row selected by embodiment_id. #pragma once @@ -35,8 +33,6 @@ struct ActionExpert { int64_t embodiment_id = 0; - // Reads ".state_enc.*", ".act_enc.*", ".act_dec.*" - // and ".pos_embd". void declare(WeightLoader & L, const char * prefix); ggml_tensor * encode_state(ggml_context * C, ggml_tensor * state) const; diff --git a/src/modules/dit_head.h b/src/modules/dit_head.h index d58a2fc..f8deaa8 100644 --- a/src/modules/dit_head.h +++ b/src/modules/dit_head.h @@ -12,14 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// DiT action head: adaLN-conditioned blocks that alternate self-attention with -// cross-attention into the vision-language embeddings. Shared by GR00T -// N1.5/N1.6/N1.7 and VLA-JEPA. -// -// A block cross-attends when `enc` is non-null and self-attends otherwise. The -// per-layer choice is the model's, since each arch interleaves differently. -// Cross-attention K/V depend only on `enc`, so a model hoists them out of the -// solver loop and passes them back through K_pre/V_pre. +// A block cross-attends when `enc` is non-null and self-attends otherwise; the +// per-layer choice is the model's. Cross-attention K/V depend only on `enc`, so +// a model hoists them out of the solver loop and passes them via K_pre/V_pre. #pragma once @@ -37,7 +32,6 @@ struct DitLayerW { ggml_tensor *Wq, *bq, *Wk, *bk, *Wv, *bv, *Wo, *bo; ggml_tensor *Wff0, *bff0, *Wff2, *bff2; - // N1.7 ships self-attention QKV and cross-attention KV pre-fused. ggml_tensor *Wqkv = nullptr, *bqkv = nullptr, *Wkv = nullptr, *bkv = nullptr; }; @@ -56,9 +50,6 @@ struct DitHead { ggml_tensor *te_l1W = nullptr, *te_l1b = nullptr, *te_l2W = nullptr, *te_l2b = nullptr; ggml_tensor *po1W = nullptr, *po1b = nullptr, *po2W = nullptr, *po2b = nullptr; - // Reads "..*", ".time_emb.*" and ".proj_out*". - // fuse_qkv builds N1.7's synthetic Wqkv (self blocks) and Wkv (cross blocks) - // by concatenating the split projections at load. void declare(WeightLoader & L, const char * prefix, bool fuse_qkv = false, bool interleave = false); void kv(ggml_context * C, const DitLayerW & w, ggml_tensor * src, @@ -69,7 +60,7 @@ struct DitHead { ggml_tensor * time_emb(ggml_context * C, ggml_tensor * tproj) const; - // Final (shift, scale) adaLN and output projection. + // (shift, scale) adaLN, opposite to layers/norm.h adaln. ggml_tensor * proj_out(ggml_context * C, ggml_tensor * h, ggml_tensor * temb) const; }; diff --git a/src/modules/encoder.h b/src/modules/encoder.h index b25280e..bd1e894 100644 --- a/src/modules/encoder.h +++ b/src/modules/encoder.h @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Post-LN transformer encoder: full self-attention with biases, a GELU MLP, and -// a LayerNorm before each residual. The SigLIP vision blocks and the GR00T -// vision-language self-attention blocks are the same computation under -// different tensor names, so both declare through EncNames. +// Post-LN encoder. The SigLIP vision blocks and the GR00T vision-language +// self-attention blocks are the same computation under different tensor names, +// which is what EncNames selects. #pragma once @@ -53,8 +52,6 @@ struct EncStack { EncCfg cfg; std::vector blk; - // Reads "..*". SigLIP checkpoints prefix their blocks with - // "blk", so callers pass e.g. "vit.blk". void declare(WeightLoader & L, const char * prefix, int64_t layers, const EncNames & n = EncNames{}); ggml_tensor * block(ggml_context * C, const EncBlockW & w, ggml_tensor * x, diff --git a/src/modules/preprocess.h b/src/modules/preprocess.h index 3e3a2b7..203a97f 100644 --- a/src/modules/preprocess.h +++ b/src/modules/preprocess.h @@ -72,8 +72,6 @@ inline bool preprocess_image_chw(const char * arch, const ImageView & v, int64_t return true; } -// HWC to a [3*ps*ps, grid*grid] patch table in [-1, 1], the GEMM patch-embed -// input GR00T N1.6 uses in place of a conv2d. inline bool preprocess_image_patches(const char * arch, const ImageView & v, int64_t side, int64_t ps, std::vector & out) { if (v.w != (int) side || v.h != (int) side || !v.data) { @@ -100,8 +98,7 @@ inline bool preprocess_image_patches(const char * arch, const ImageView & v, int return true; } -// Pixel shuffle with c-outermost channel order, the inverse layout to -// pixel_shuffle_hf above. GR00T N1.6's connector expects this one. +// c-outermost channel order, the inverse layout to pixel_shuffle_hf above. inline void pixel_shuffle_back(const float * src, int64_t grid, int64_t hidden, int64_t r, float * dst) { const int64_t g2 = grid/r, c4 = hidden*r*r; for (int64_t y = 0; y < g2; ++y) diff --git a/src/modules/prompt.h b/src/modules/prompt.h index 43f9a16..97c5168 100644 --- a/src/modules/prompt.h +++ b/src/modules/prompt.h @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Token-sequence assembly for the archs whose backbone takes one interleaved -// image/text stream: build the id list, note where the image slots landed, then -// swap the tower's embeddings into those rows. +// Token-sequence assembly for backbones taking one interleaved image/text +// stream. #pragma once @@ -35,18 +34,14 @@ struct Prompt { int64_t n_text() const { return (int64_t) text_pos.size(); } }; -// Accepts a caller-supplied stream that already carries exactly n_img image -// placeholders, or one with none, in which case the placeholders are prepended. -// Any other count is a mismatch between the tokenizer and the tower. +// Accepts a stream carrying exactly n_img placeholders, or none, in which case +// they are prepended. bool build_prompt(const char * arch, const Inputs & in, int64_t n_img, int32_t image_token, int64_t max_seq, Prompt & out); -// Embedding-table rows for the prompt, with the image rows overwritten by the -// tower output. bool fetch_embeds(const char * arch, gguf_reader & io, const Prompt & p, const float * img_emb, int64_t hidden, std::vector & out); -// The request's noise if it carried any, else a fresh N(0,1) draw. void init_noise(const Inputs & in, size_t n, std::vector & out); } diff --git a/src/modules/qwen3_lm.h b/src/modules/qwen3_lm.h index 3ee23b4..6226213 100644 --- a/src/modules/qwen3_lm.h +++ b/src/modules/qwen3_lm.h @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Qwen3 decoder stack, shared by GR00T N1.5/N1.6/N1.7 and VLA-JEPA. Prefill -// only: a VLA runs one pass per action chunk, so there is no KV cache. +// Prefill only: a VLA runs one pass per action chunk, so there is no KV cache. #pragma once @@ -48,13 +47,11 @@ struct Qwen3LM { std::vector blk; ggml_tensor * output_norm = nullptr; - // Reads ".blk..*" and ".output_norm.weight". void declare(WeightLoader & L, const char * prefix); ggml_tensor * block(ggml_context * C, const Qwen3LayerW & w, ggml_tensor * h, ggml_tensor * pos, ggml_tensor * mask, int64_t seq) const; - // Every block, then the output RMSNorm. ggml_tensor * build(ggml_context * C, ggml_tensor * h, ggml_tensor * pos, ggml_tensor * mask, int64_t seq) const; }; diff --git a/src/modules/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h index b53cbb5..bad980b 100644 --- a/src/modules/qwen3vl_vit.h +++ b/src/modules/qwen3vl_vit.h @@ -19,7 +19,7 @@ #include "model.h" #include "ggml.h" -#include "env_flag.h" +#include "options.h" #include #include @@ -44,8 +44,6 @@ inline ggml_tensor * rope2d(ggml_context * C, ggml_tensor * x, ggml_tensor * cos return ggml_add(C, ggml_mul(C, x, cos_t), ggml_mul(C, rot, sin_t)); } -inline bool fa_enabled() { static const bool e = vla::env_flag("VLA_FLASH_ATTN"); return e; } - inline ggml_tensor * flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * mask, float scale) { ggml_tensor * kf = (k->type == GGML_TYPE_F16) ? k : ggml_cast(C, k, GGML_TYPE_F16); @@ -68,7 +66,7 @@ inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, seq), 0, 2, 1, 3)); Q = rope2d(C, Q, cos_t, sin_t); K = rope2d(C, K, cos_t, sin_t); ggml_tensor * att; - if (fa_enabled()) { + if (vla::flash_attn_enabled()) { ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 0, 2, 1, 3)); att = flash_attn(C, Q, K, V, nullptr, scale); } else { diff --git a/src/modules/siglip_vit.h b/src/modules/siglip_vit.h index 1a88bb9..9ac91ca 100644 --- a/src/modules/siglip_vit.h +++ b/src/modules/siglip_vit.h @@ -12,12 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// SigLIP vision tower: learned position embeddings, post-LN encoder blocks, a -// final LayerNorm, and no CLS token. -// -// The patch embedding has two spellings. GR00T N1.5, pi0, pi0.5 and SmolVLA -// feed a CHW image through a conv2d; N1.6 pre-patchifies on the host and feeds -// a GEMM. Both land on the same [hidden, patches] activation. +// The patch embedding has two spellings: a conv2d over a CHW image, or a GEMM +// over host-patchified input. Both land on the same [hidden, patches]. #pragma once @@ -35,14 +31,11 @@ struct SigLipTower { ggml_tensor *pos = nullptr; ggml_tensor *post_ln_w = nullptr, *post_ln_b = nullptr; - // Reads ".patch_embd.*", ".pos_embd", ".post_ln.*" - // and ".blk..*". void declare(WeightLoader & L, const char * prefix, int64_t layers, bool patch_embd_is_gemm = false); ggml_tensor * embed_conv(ggml_context * C, ggml_tensor * pixels, int64_t patch, int64_t grid) const; ggml_tensor * embed_patches(ggml_context * C, ggml_tensor * patches) const; - // Encoder blocks, then the final LayerNorm. ggml_tensor * build(ggml_context * C, ggml_tensor * h, int64_t seq, int64_t nv = 1) const; }; diff --git a/src/options.cpp b/src/options.cpp new file mode 100644 index 0000000..b4fa63f --- /dev/null +++ b/src/options.cpp @@ -0,0 +1,193 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "options.h" + +#include "nlohmann/json.hpp" + +#include +#include +#include +#include + +namespace vla { + +namespace { + +bool parse_dtype(const std::string & v, ggml_type & out) { + if (v == "f32" || v == "fp32") { out = GGML_TYPE_F32; return true; } + if (v == "bf16" || v == "bfp16") { out = GGML_TYPE_BF16; return true; } + return false; +} + +bool parse_bool(const std::string & v, bool & out) { + if (v == "1" || v == "true" || v == "on" || v == "yes") { out = true; return true; } + if (v == "0" || v == "false" || v == "off" || v == "no") { out = false; return true; } + return false; +} + +bool parse_int(const std::string & v, int & out) { + char * end = nullptr; + const long n = std::strtol(v.c_str(), &end, 10); + if (end == v.c_str() || *end) return false; + out = (int) n; + return true; +} + +} + +const char * dtype_name(ggml_type t) { + return t == GGML_TYPE_BF16 ? "bf16" : "f32"; +} + +namespace { +bool g_flash_attn = false; +bool g_mm_prec_f32 = true; +} + +void set_flash_attn(bool on) { g_flash_attn = on; } +bool flash_attn_enabled() { return g_flash_attn; } + +void set_mm_prec_f32(bool on) { g_mm_prec_f32 = on; } +bool mm_prec_f32_enabled() { return g_mm_prec_f32; } + +const char * Options::usage() { + return " --weight-dtype f32|bf16 resident dtype for GEMM weights\n" + " --act-dtype f32|bf16 activation dtype (needs CUDA and bf16 weights)\n" + " --flash-attn [0|1] flash attention; faster, changes numerics\n" + " --mm-prec default|f32 matmul accumulation precision\n" + " --n-threads N CPU threads for the in-tree loaders\n" + " --num-steps N flow-matching solver steps\n" + " --embodiment TAG GR00T embodiment tag or id\n" + " --unnorm-key KEY un-normalisation statistics key\n"; +} + +bool Options::parse_arg(int argc, char ** argv, int & i, std::string & err) { + const std::string a = argv[i]; + + auto next = [&](std::string & v) -> bool { + if (i+1 >= argc) { err = a+" needs a value"; return false; } + v = argv[++i]; + return true; + }; + + if (a == "--weight-dtype" || a == "--act-dtype") { + std::string v; + if (!next(v)) return false; + + ggml_type t; + if (!parse_dtype(v, t)) { err = a+": expected f32 or bf16, got '"+v+"'"; return false; } + if (a == "--weight-dtype") weight_dtype = t; + else act_dtype = t; + return true; + } + + if (a == "--flash-attn") { + bool v = true; + if (i+1 < argc && argv[i+1][0] != '-' && parse_bool(argv[i+1], v)) ++i; + flash_attn = v; + return true; + } + + if (a == "--mm-prec") { + std::string v; + if (!next(v)) return false; + if (v == "default") { mm_prec_f32 = false; return true; } + if (v == "f32") { mm_prec_f32 = true; return true; } + err = "--mm-prec: expected default or f32, got '"+v+"'"; + return false; + } + + if (a == "--n-threads" || a == "--num-steps") { + std::string v; + if (!next(v)) return false; + + int n = 0; + if (!parse_int(v, n) || n <= 0) { err = a+": expected a positive integer, got '"+v+"'"; return false; } + if (a == "--n-threads") n_threads = n; + else num_steps = n; + return true; + } + + if (a == "--embodiment") { std::string v; if (!next(v)) return false; embodiment = v; return true; } + if (a == "--unnorm-key") { std::string v; if (!next(v)) return false; unnorm_key = v; return true; } + + err.clear(); + return false; +} + +// These moved to CLI flags. Leaving them silently ignored would let a +// benchmark script measure a configuration it did not ask for. +bool Options::reject_retired_env(std::string & err) { + static const char * const retired[][2] = { + {"VLA_GR00T_BF16_WEIGHTS", "--weight-dtype bf16"}, + {"VLA_JEPA_BF16_WEIGHTS", "--weight-dtype bf16"}, + {"VLA_BITVLA_BF16_WEIGHTS", "--weight-dtype bf16"}, + {"VLA_PI0_F32_WEIGHTS", "--weight-dtype f32"}, + {"VLA_PI05_F32_WEIGHTS", "--weight-dtype f32"}, + {"VLA_EVO1_F32_WEIGHTS", "--weight-dtype f32"}, + {"VLA_ADAPTER_F32_WEIGHTS", "--weight-dtype f32"}, + {"VLA_OPENVLA_OFT_F32_WEIGHTS","--weight-dtype f32"}, + {"VLA_PI0_BF16_ACT", "--act-dtype bf16"}, + {"VLA_EVO1_BF16_ACT", "--act-dtype bf16"}, + {"VLA_SMOLVLA_FA", "--flash-attn"}, + {"VLA_PI0_FA", "--flash-attn"}, + {"VLA_EVO1_FA", "--flash-attn"}, + {"VLA_FLASH_ATTN", "--flash-attn"}, + {"VLA_MM_PREC", "--mm-prec"}, + {"VLA_WEIGHT_DTYPE", "--weight-dtype"}, + }; + + for (const auto & r : retired) + if (std::getenv(r[0])) { + err = std::string(r[0])+" is no longer read; pass "+r[1]+" instead"; + return false; + } + return true; +} + +bool Options::load_json(const std::string & path, std::string & err) { + if (path.empty()) return true; + + std::ifstream f(path); + if (!f) return true; + + nlohmann::json j; + try { + f >> j; + } catch (const std::exception & e) { + err = std::string("config json: ")+e.what(); + return false; + } + if (!j.contains("runtime") || !j["runtime"].is_object()) return true; + const nlohmann::json & r = j["runtime"]; + + try { + ggml_type t; + if (r.contains("weight_dtype") && parse_dtype(r["weight_dtype"].get(), t)) weight_dtype = t; + if (r.contains("act_dtype") && parse_dtype(r["act_dtype"].get(), t)) act_dtype = t; + if (r.contains("flash_attn")) flash_attn = r["flash_attn"].get(); + if (r.contains("mm_prec")) mm_prec_f32 = r["mm_prec"].get() == "f32"; + if (r.contains("n_threads")) n_threads = r["n_threads"].get(); + if (r.contains("num_steps")) num_steps = r["num_steps"].get(); + if (r.contains("embodiment")) embodiment = r["embodiment"].get(); + if (r.contains("unnorm_key")) unnorm_key = r["unnorm_key"].get(); + } catch (const std::exception & e) { + err = std::string("config json runtime: ")+e.what(); + return false; + } + return true; +} + +} diff --git a/src/options.h b/src/options.h new file mode 100644 index 0000000..2119500 --- /dev/null +++ b/src/options.h @@ -0,0 +1,66 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Runtime knobs that trade precision for speed. Every field is unset by +// default; a model reads one with value_or() so its own default stays at its +// own call site, and both branches are always compiled. +// +// The fastest setting differs per architecture and some of them change +// numerics, so none of this can be decided at build time. + +#pragma once + +#include "ggml.h" + +#include +#include + +namespace vla { + +struct Options { + std::optional weight_dtype; + std::optional act_dtype; + std::optional flash_attn; + std::optional mm_prec_f32; + std::optional n_threads; + std::optional num_steps; + std::optional embodiment; + std::optional unnorm_key; + + // Consumes argv[i] (and its value) if it names an option. Returns false + // with err set on a bad value; leaves i untouched and err empty when the + // argument is not ours. + bool parse_arg(int argc, char ** argv, int & i, std::string & err); + + // Fails if a caller still sets one of the env switches these replaced. + static bool reject_retired_env(std::string & err); + + // Merges the "runtime" object of a policy config.json, if present. + bool load_json(const std::string & path, std::string & err); + + static const char * usage(); +}; + +const char * dtype_name(ggml_type t); + +// Flash attention is decided once per loaded model but read from graph builders +// several call levels down, so it is held here rather than threaded through +// every signature. +void set_flash_attn(bool on); +bool flash_attn_enabled(); + +void set_mm_prec_f32(bool on); +bool mm_prec_f32_enabled(); + +} diff --git a/src/serving/server.cpp b/src/serving/server.cpp index 63bcb23..26790a0 100644 --- a/src/serving/server.cpp +++ b/src/serving/server.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "model.h" +#include "options.h" #include "serving/hf_fetch.h" #include "serving/vla.pb.h" @@ -190,10 +191,11 @@ void usage(const char * prog) { " 'none' : single ms_inference\n" " 'phase' : ms_prefill + ms_denoise broken out\n" " (π0 currently reports only the combined ms_inference)\n" + "%s" " --config PATH LeRobot policy config.json (SmolVLA safetensors only;\n" " ignored for GGUF checkpoints). If omitted, uses\n" " /config.json.\n", - prog); + prog, vla::Options::usage()); } } @@ -209,6 +211,8 @@ int main(int argc, char ** argv) { std::string hf_spec; std::string config_path; vla::TimingDetail timing_detail = vla::TimingDetail::NONE; + vla::Options opts; + std::string opt_err; std::vector positionals; for (int i = 1; i < argc; ++i) { @@ -228,6 +232,12 @@ int main(int argc, char ** argv) { usage(argv[0]); return 1; } + } else if (opts.parse_arg(argc, argv, i, opt_err)) { + continue; + } else if (!opt_err.empty()) { + std::fprintf(stderr, "vla-server: %s\n", opt_err.c_str()); + usage(argv[0]); + return 1; } else if (a == "-h" || a == "--help") { usage(argv[0]); return 0; @@ -261,7 +271,12 @@ int main(int argc, char ** argv) { if (!config_path.empty()) { std::printf(" config: %s\n", config_path.c_str()); } - vla::Model * model = vla::model_load(mmproj_path, ckpt_path, config_path); + if (!opts.load_json(config_path, opt_err)) { + std::fprintf(stderr, "vla-server: %s\n", opt_err.c_str()); + return 1; + } + + vla::Model * model = vla::model_load(mmproj_path, ckpt_path, config_path, opts); if (!model) { std::fprintf(stderr, "vla-server: model_load failed\n"); return 1; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5e60cc4..cb9782a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,11 +7,13 @@ target_compile_options(vla_predict_check PRIVATE -Wall -Wextra) add_executable(test_vision_common test_vision_common.cpp) target_include_directories(test_vision_common PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(test_vision_common PRIVATE ggml) target_compile_options(test_vision_common PRIVATE -Wall -Wextra) add_test(NAME vision_common COMMAND test_vision_common) add_executable(test_rope_conventions test_rope_conventions.cpp) target_include_directories(test_rope_conventions PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(test_rope_conventions PRIVATE ggml) target_compile_options(test_rope_conventions PRIVATE -Wall -Wextra) add_test(NAME rope_conventions COMMAND test_rope_conventions) diff --git a/tests/predict_check.cpp b/tests/predict_check.cpp index f27f7f0..e7c425b 100644 --- a/tests/predict_check.cpp +++ b/tests/predict_check.cpp @@ -22,6 +22,7 @@ // VLA_TIMING=phase, VLA_EXTRA_TOKEN / VLA_EXTRA_COUNT #include "model.h" +#include "options.h" #include #include @@ -34,14 +35,25 @@ using namespace vla; int main(int argc, char** argv) { if (argc < 2) { - std::fprintf(stderr, "usage: %s [mmproj.gguf] [n_images]\n", argv[0]); + std::fprintf(stderr, "usage: %s [mmproj.gguf] [n_images] [options]\n%s", argv[0], Options::usage()); return 1; } const char* ckpt = argv[1]; const char* mmproj = (argc > 2 && argv[2][0] && argv[2][0] != '-') ? argv[2] : ""; - const int n_images = argc > 3 ? std::atoi(argv[3]) : 2; + const int n_images = (argc > 3 && argv[3][0] != '-') ? std::atoi(argv[3]) : 2; - Model* m = model_load(mmproj, ckpt, ""); + Options opts; + for (int i = 2; i < argc; ++i) { + if (argv[i][0] != '-') continue; + std::string err; + if (!opts.parse_arg(argc, argv, i, err)) { + if (!err.empty()) { std::fprintf(stderr, "%s\n", err.c_str()); return 1; } + std::fprintf(stderr, "unknown option %s\n", argv[i]); + return 1; + } + } + + Model* m = model_load(mmproj, ckpt, "", opts); if (!m) { std::fprintf(stderr, "model_load failed\n"); return 1; From f787c6e474cac72647d230415a721959fc066c05 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 22:17:37 +0700 Subject: [PATCH 08/21] fold the qwen3-vl tower's private rope and attention onto the layer primitives --- src/models/gr00tn1d7.cpp | 27 +++++++-------------------- src/modules/qwen3vl_vit.h | 35 ++++++----------------------------- 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 8862385..6d9332c 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "layers/attn.h" #include "options.h" #include "model.h" @@ -134,18 +135,11 @@ ggml_tensor * build_vlsa_layer(ggml_context * C, const VlsaLayerW & w, ggml_tens ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, n1), w.bv); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, seq), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, seq), 0, 2, 1, 3)); + ggml_tensor * Q = to_heads(C, q, hd, heads, seq); + ggml_tensor * K = to_heads(C, k, hd, heads, seq); ggml_tensor * att; - if (vla::flash_attn_enabled()) { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 0, 2, 1, 3)); - att = flash_attn(C, Q, K, V, nullptr, scale); - } else { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq); - } + if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, to_heads (C, v, hd, heads, seq), nullptr, scale); + else att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n3 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.n3w), w.n3b); ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); @@ -171,15 +165,8 @@ ggml_tensor * build_qwen3_layer(ggml_context * C, const Gr00tN1d7ModelArch & m, ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); ggml_tensor * att; - if (vla::flash_attn_enabled()) { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 0, 2, 1, 3)); - att = flash_attn(C, Q, K, V, ggml_cast(C, mask, GGML_TYPE_F16), scale); - } else { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, mask, scale, 0.0f); - att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hq, seq); - } + if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, ggml_cont(C, ggml_permute(C, vh, 0, 2, 1, 3)), ggml_cast(C, mask, GGML_TYPE_F16), scale); + else att = attention (C, Q, K, ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)), mask, scale, hq, seq); ggml_tensor * h_attn = ggml_add(C, h, ggml_mul_mat(C, w.Wo, att)); ggml_tensor * hn2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); ggml_tensor * gate = ggml_silu(C, ggml_mul_mat(C, w.Wgate, hn2)); diff --git a/src/modules/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h index bad980b..316a8d1 100644 --- a/src/modules/qwen3vl_vit.h +++ b/src/modules/qwen3vl_vit.h @@ -16,6 +16,8 @@ #pragma once +#include "layers/attn.h" +#include "layers/rope.h" #include "model.h" #include "ggml.h" @@ -36,23 +38,6 @@ constexpr float QWEN3VL_STD [3] = {0.5f, 0.5f, 0.5f}; struct VitLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wqkv,*bqkv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; struct MergerW { ggml_tensor *nw,*nb,*fc1w,*fc1b,*fc2w,*fc2b; }; -inline ggml_tensor * rope2d(ggml_context * C, ggml_tensor * x, ggml_tensor * cos_t, ggml_tensor * sin_t) { - const int64_t hd = x->ne[0], S = x->ne[1], Hh = x->ne[2]; const int64_t half = hd / 2; - ggml_tensor * x1 = ggml_cont(C, ggml_view_3d(C, x, half, S, Hh, x->nb[1], x->nb[2], 0)); - ggml_tensor * x2 = ggml_cont(C, ggml_view_3d(C, x, half, S, Hh, x->nb[1], x->nb[2], (size_t) half * x->nb[0])); - ggml_tensor * rot = ggml_concat(C, ggml_neg(C, x2), x1, 0); - return ggml_add(C, ggml_mul(C, x, cos_t), ggml_mul(C, rot, sin_t)); -} - -inline ggml_tensor * flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, - ggml_tensor * mask, float scale) { - ggml_tensor * kf = (k->type == GGML_TYPE_F16) ? k : ggml_cast(C, k, GGML_TYPE_F16); - ggml_tensor * vf = (v->type == GGML_TYPE_F16) ? v : ggml_cast(C, v, GGML_TYPE_F16); - ggml_tensor * o = ggml_flash_attn_ext(C, q, kf, vf, mask, scale, 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32); - return ggml_reshape_2d(C, o, o->ne[0] * o->ne[1], o->ne[2] * o->ne[3]); -} - inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml_tensor * x, ggml_tensor * cos_t, ggml_tensor * sin_t, int64_t seq, int64_t heads, int64_t hd, int64_t hidden, float ln_eps) { @@ -62,19 +47,11 @@ inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml ggml_tensor * q = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], 0)); ggml_tensor * k = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], (size_t) hidden * qkv->nb[0])); ggml_tensor * v = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], (size_t) 2 * hidden * qkv->nb[0])); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, seq), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, seq), 0, 2, 1, 3)); - Q = rope2d(C, Q, cos_t, sin_t); K = rope2d(C, K, cos_t, sin_t); + ggml_tensor * Q = rope_2d(C, to_heads(C, q, hd, heads, seq), cos_t, sin_t); + ggml_tensor * K = rope_2d(C, to_heads(C, k, hd, heads, seq), cos_t, sin_t); ggml_tensor * att; - if (vla::flash_attn_enabled()) { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 0, 2, 1, 3)); - att = flash_attn(C, Q, K, V, nullptr, scale); - } else { - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, seq), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq); - } + if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, to_heads (C, v, hd, heads, seq), nullptr, scale); + else att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); From 69e908c4284be4098872d24edb5d062bfa195aab Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 22:24:09 +0700 Subject: [PATCH 09/21] put gr00t n1.7 and vla-jepa on the shared tower, backbone and dit modules --- src/models/gr00tn1d7.cpp | 336 +++++++++----------------------------- src/models/vla_jepa.cpp | 213 +++++++----------------- src/modules/dit_head.cpp | 20 ++- src/modules/dit_head.h | 5 +- src/modules/qwen3vl_vit.h | 50 ++++++ 5 files changed, 207 insertions(+), 417 deletions(-) diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 6d9332c..4a0eb89 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -14,6 +14,12 @@ #include "arch.h" #include "layers/attn.h" +#include "layers/linear.h" +#include "layers/norm.h" +#include "modules/action_expert.h" +#include "modules/dit_head.h" +#include "modules/encoder.h" +#include "modules/qwen3_lm.h" #include "options.h" #include "model.h" @@ -24,7 +30,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "modules/qwen3vl_vit.h" #include "env_flag.h" @@ -77,21 +83,12 @@ struct Gr00tN1d7ModelArch : public ModelArchBase { float vlln_eps=1e-5f, vlsa_ln_eps=1e-5f, ln_eps=1e-5f, norm_out_eps=1e-6f, connector_ln_eps=1e-6f; int64_t embodiment_id = 2; - ggml_tensor *vit_patch_w=nullptr,*vit_patch_b=nullptr,*vit_pos=nullptr; - std::vector vit; - MergerW deepstack[3]; - MergerW merger; - ggml_tensor *lm_output_norm=nullptr; - std::vector lm; + Qwen3VLTower vit; + Qwen3LM lm; + EncStack vlsa; + ActionExpert aex; + DitHead dit; ggml_tensor *vlln_w=nullptr,*vlln_b=nullptr; - std::vector vlsa; - ggml_tensor *se_l1W=nullptr,*se_l1b=nullptr,*se_l2W=nullptr,*se_l2b=nullptr; - ggml_tensor *ae_W1W=nullptr,*ae_W1b=nullptr,*ae_W2W=nullptr,*ae_W2b=nullptr,*ae_W3W=nullptr,*ae_W3b=nullptr; - ggml_tensor *ad_l1W=nullptr,*ad_l1b=nullptr,*ad_l2W=nullptr,*ad_l2b=nullptr; - ggml_tensor *pos_embd=nullptr; - ggml_tensor *te_l1W=nullptr,*te_l1b=nullptr,*te_l2W=nullptr,*te_l2b=nullptr; - std::vector dit; - ggml_tensor *po1W=nullptr,*po1b=nullptr,*po2W=nullptr,*po2b=nullptr; bool caches_ready = false; std::vector c_grow, c_gcol; @@ -128,93 +125,9 @@ ggml_tensor * head_view(ggml_context * C, ggml_tensor * proj, int64_t hd, int64_ return ggml_view_3d(C, proj, hd, heads, T, (size_t) hd * es, (size_t) nblk * E * es, (size_t) blk * E * es); } -ggml_tensor * build_vlsa_layer(ggml_context * C, const VlsaLayerW & w, ggml_tensor * x, - int64_t seq, int64_t heads, int64_t hd, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.n1w), w.n1b); - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, n1), w.bv); - ggml_tensor * Q = to_heads(C, q, hd, heads, seq); - ggml_tensor * K = to_heads(C, k, hd, heads, seq); - ggml_tensor * att; - if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, to_heads (C, v, hd, heads, seq), nullptr, scale); - else att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); - ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.n3w), w.n3b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} -ggml_tensor * build_qwen3_layer(ggml_context * C, const Gr00tN1d7ModelArch & m, const Qwen3LayerW & w, - ggml_tensor * h, ggml_tensor * positions, ggml_tensor * mask, int64_t seq) { - const int64_t hd = m.lm_head_dim, n_q = m.n_q, n_kv = m.n_kv, hq = n_q * hd; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * hn = ggml_mul(C, ggml_rms_norm(C, h, m.lm_rms_eps), w.attn_norm); - ggml_tensor * qp = ggml_mul_mat(C, w.Wq, hn); - ggml_tensor * kp = ggml_mul_mat(C, w.Wk, hn); - ggml_tensor * vp = ggml_mul_mat(C, w.Wv, hn); - ggml_tensor * qh = ggml_reshape_3d(C, qp, hd, n_q, seq); - ggml_tensor * kh = ggml_reshape_3d(C, kp, hd, n_kv, seq); - ggml_tensor * vh = ggml_reshape_3d(C, vp, hd, n_kv, seq); - ggml_tensor * qn = ggml_mul(C, ggml_rms_norm(C, qh, m.lm_rms_eps), w.q_norm); - ggml_tensor * kn = ggml_mul(C, ggml_rms_norm(C, kh, m.lm_rms_eps), w.k_norm); - int sections[4] = { 24, 20, 20, 0 }; - ggml_tensor * qr = ggml_rope_multi(C, qn, positions, nullptr, (int) hd, sections, GGML_ROPE_TYPE_IMROPE, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * kr = ggml_rope_multi(C, kn, positions, nullptr, (int) hd, sections, GGML_ROPE_TYPE_IMROPE, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); - ggml_tensor * att; - if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, ggml_cont(C, ggml_permute(C, vh, 0, 2, 1, 3)), ggml_cast(C, mask, GGML_TYPE_F16), scale); - else att = attention (C, Q, K, ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)), mask, scale, hq, seq); - ggml_tensor * h_attn = ggml_add(C, h, ggml_mul_mat(C, w.Wo, att)); - ggml_tensor * hn2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); - ggml_tensor * gate = ggml_silu(C, ggml_mul_mat(C, w.Wgate, hn2)); - ggml_tensor * up = ggml_mul_mat(C, w.Wup, hn2); - return ggml_add(C, h_attn, ggml_mul_mat(C, w.Wdown, ggml_mul(C, gate, up))); -} -void dit_kv(ggml_context * C, const Gr00tN1d7ModelArch & m, const DitLayerW & w, ggml_tensor * kv, - ggml_tensor ** K_out, ggml_tensor ** V_out) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, dim = m.dit_hidden, Tkv = kv->ne[1]; - if (w.Wkv) { - ggml_tensor * kvp = ggml_add(C, ggml_mul_mat(C, w.Wkv, kv), w.bkv); - *K_out = ggml_cont(C, ggml_permute(C, head_view(C, kvp, hd, heads, Tkv, dim, 2, 0), 0, 2, 1, 3)); - *V_out = ggml_cont(C, ggml_permute(C, head_view(C, kvp, hd, heads, Tkv, dim, 2, 1), 1, 2, 0, 3)); - return; - } - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, kv), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, kv), w.bv); - *K_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, Tkv), 0, 2, 1, 3)); - *V_out = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, Tkv), 1, 2, 0, 3)); -} -ggml_tensor * build_dit_block(ggml_context * C, const Gr00tN1d7ModelArch & m, const DitLayerW & w, - ggml_tensor * h, ggml_tensor * temb, ggml_tensor * enc , - ggml_tensor * K_pre = nullptr, ggml_tensor * V_pre = nullptr) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, dim = m.dit_hidden, Tk = h->ne[1]; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * n = adaln(C, h, temb, w.adaln_w, w.adaln_b, dim, m.ln_eps); - ggml_tensor * K, * V, * Q; - if (!enc && w.Wqkv) { - ggml_tensor * qkv = ggml_add(C, ggml_mul_mat(C, w.Wqkv, n), w.bqkv); - Q = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 0), 0, 2, 1, 3)); - K = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 1), 0, 2, 1, 3)); - V = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 2), 1, 2, 0, 3)); - } else { - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n), w.bq); - Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, Tk), 0, 2, 1, 3)); - if (K_pre) { K = K_pre; V = V_pre; } - else { dit_kv(C, m, w, enc ? enc : n, &K, &V); } - } - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), dim, Tk); - ggml_tensor * h1 = ggml_add(C, h, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_norm(C, h1, m.ln_eps); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; @@ -251,7 +164,36 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { F(fk("vlln_eps"), m.vlln_eps); F(fk("vlsa_ln_eps"), m.vlsa_ln_eps); F(fk("connector_ln_eps"), m.connector_ln_eps); F(fk("vit_rope_theta"), m.vit_rope_base); if (g.has(fk("lm_rope_theta"))) m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); - m.embodiment_id = 2; + m.lm.cfg.hidden = m.lm_hidden; + m.lm.cfg.layers = m.lm_layers; + m.lm.cfg.n_q = m.n_q; + m.lm.cfg.n_kv = m.n_kv; + m.lm.cfg.head_dim = m.lm_head_dim; + m.lm.cfg.inter = m.lm_inter; + m.lm.cfg.rms_eps = m.lm_rms_eps; + m.lm.cfg.flash_attn = flash_attn_enabled(); + m.lm.cfg.rope.type = GGML_ROPE_TYPE_IMROPE; + m.lm.cfg.rope.n_dims = (int) m.lm_head_dim; + m.lm.cfg.rope.freq_base = m.lm_rope_base; + m.lm.cfg.rope.sections[0]= 24; + m.lm.cfg.rope.sections[1]= 20; + m.lm.cfg.rope.sections[2]= 20; + m.lm.cfg.rope.sections[3]= 0; + + m.vlsa.cfg.hidden = m.bb_embed_dim; + m.vlsa.cfg.heads = m.vlsa_heads; + m.vlsa.cfg.head_dim = m.vlsa_head_dim; + m.vlsa.cfg.ln_eps = m.vlsa_ln_eps; + m.vlsa.cfg.flash_attn = flash_attn_enabled(); + + m.dit.cfg.hidden = m.dit_hidden; + m.dit.cfg.heads = m.dit_heads; + m.dit.cfg.head_dim = m.dit_head_dim; + m.dit.cfg.layers = m.dit_layers; + m.dit.cfg.ln_eps = m.ln_eps; + m.dit.cfg.norm_out_eps = m.norm_out_eps; + + m.aex.embodiment_id = 2; { const std::string js = g.str(fk("embodiment_id_mapping")); auto lookup = [&](const char * key) -> long { @@ -260,14 +202,14 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { p = js.find(':', p + k.size()); if (p == std::string::npos) return -1; return std::strtol(js.c_str() + p + 1, nullptr, 10); }; - long ls = lookup("libero_sim"); if (ls >= 0) m.embodiment_id = ls; + long ls = lookup("libero_sim"); if (ls >= 0) m.aex.embodiment_id = ls; if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { char * end = nullptr; long v = std::strtol(e, &end, 10); - if (end && *end == '\0') m.embodiment_id = v; - else { long id = lookup(e); if (id >= 0) m.embodiment_id = id; else std::fprintf(stderr, "vla(gr00tn1d7): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.embodiment_id); } + if (end && *end == '\0') m.aex.embodiment_id = v; + else { long id = lookup(e); if (id >= 0) m.aex.embodiment_id = id; else std::fprintf(stderr, "vla(gr00tn1d7): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); } } } - if (m.embodiment_id < 0 || m.embodiment_id >= m.max_embodiments) { std::fprintf(stderr, "vla(gr00tn1d7): embodiment id %lld out of range [0,%lld)\n", (long long) m.embodiment_id, (long long) m.max_embodiments); return false; } + if (m.aex.embodiment_id < 0 || m.aex.embodiment_id >= m.max_embodiments) { std::fprintf(stderr, "vla(gr00tn1d7): embodiment id %lld out of range [0,%lld)\n", (long long) m.aex.embodiment_id, (long long) m.max_embodiments); return false; } cfg = Config{}; cfg.n_img = 64; cfg.n_lang = m.max_seq_len; cfg.n_state = 1; @@ -315,7 +257,7 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, (long long) m->lm_hidden, (long long) m->lm_layers, (long long) m->n_q, (long long) m->n_kv, (long long) m->lm_head_dim, (double) m->lm_rope_base, (long long) m->vlsa_layers, (long long) m->vlsa_heads, (long long) m->vlsa_head_dim, (long long) m->dit_layers, (long long) m->dit_heads, (long long) m->dit_head_dim, (long long) m->dit_hidden, (long long) m->attend_text_every_n, (long long) m->in_embed_dim, - (long long) m->action_horizon, (long long) m->action_dim, (long long) m->max_state_dim, (long long) m->num_steps, (long long) m->embodiment_id, + (long long) m->action_horizon, (long long) m->action_dim, (long long) m->max_state_dim, (long long) m->num_steps, (long long) m->aex.embodiment_id, m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); { @@ -324,146 +266,28 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, m->backend = b.handle; } - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d7): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(gr00tn1d7): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; - - struct FusedSpec { ggml_tensor * dst; std::vector srcs; }; - std::vector fused; - const bool fuse = true; - auto mk_fused = [&](const char * out_name, std::vector srcs, ggml_type type) -> ggml_tensor * { - const ggml_tensor * g0 = g.meta(srcs[0]); - if (!g0) { std::fprintf(stderr, "vla(gr00tn1d7): fused src missing %s\n", srcs[0]); return nullptr; } - const bool is1d = ggml_n_dims(g0) == 1; - int64_t ne0 = g0->ne[0], acc = 0; - for (const char * s : srcs) { const ggml_tensor * gs = g.meta(s); if (!gs) { std::fprintf(stderr, "vla(gr00tn1d7): fused src missing %s\n", s); return nullptr; } acc += is1d ? gs->ne[0] : gs->ne[1]; } - ggml_tensor * t = is1d ? ggml_new_tensor_1d(W, type, acc) : ggml_new_tensor_2d(W, type, ne0, acc); - ggml_set_name(t, out_name); - FusedSpec fs{t, {}}; for (const char * s : srcs) fs.srcs.emplace_back(s); fused.push_back(std::move(fs)); - return t; - }; - bool ok = true; - - m->vit_patch_w = mk_mm("vit.patch_embd.weight"); m->vit_patch_b = mk_f32("vit.patch_embd.bias"); m->vit_pos = mk_f32("vit.pos_embd"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wqkv=mk_mm(N("attn_qkv.weight")); w.bqkv=mk_f32(N("attn_qkv.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - ok &= w.ln1w&&w.ln1b&&w.ln2w&&w.ln2b&&w.Wqkv&&w.bqkv&&w.Wo&&w.bo&&w.Wfc1&&w.bfc1&&w.Wfc2&&w.bfc2; - } - for (int j = 0; j < 3; ++j) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.deepstack.%d.%s", j, s); return p; }; - auto & w = m->deepstack[j]; - w.nw=mk_f32(N("norm.weight")); w.nb=mk_f32(N("norm.bias")); w.fc1w=mk_mm(N("fc1.weight")); w.fc1b=mk_f32(N("fc1.bias")); w.fc2w=mk_mm(N("fc2.weight")); w.fc2b=mk_f32(N("fc2.bias")); - ok &= w.nw&&w.nb&&w.fc1w&&w.fc1b&&w.fc2w&&w.fc2b; - } - { auto & w = m->merger; - w.nw=mk_f32("vit.merger.norm.weight"); w.nb=mk_f32("vit.merger.norm.bias"); w.fc1w=mk_mm("vit.merger.fc1.weight"); w.fc1b=mk_f32("vit.merger.fc1.bias"); w.fc2w=mk_mm("vit.merger.fc2.weight"); w.fc2b=mk_f32("vit.merger.fc2.bias"); - ok &= w.nw&&w.nb&&w.fc1w&&w.fc1b&&w.fc2w&&w.fc2b; } - - m->lm_output_norm = mk_f32("vlm.output_norm.weight"); - m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vlm.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->lm[i]; - w.attn_norm=mk_f32(N("attn_norm.weight")); - w.Wq=mk_mm(N("attn_q.weight")); w.Wk=mk_mm(N("attn_k.weight")); w.Wv=mk_mm(N("attn_v.weight")); w.Wo=mk_mm(N("attn_o.weight")); - w.q_norm=mk_f32(N("attn_q_norm.weight")); w.k_norm=mk_f32(N("attn_k_norm.weight")); w.ffn_norm=mk_f32(N("ffn_norm.weight")); - w.Wgate=mk_mm(N("ffn_gate.weight")); w.Wup=mk_mm(N("ffn_up.weight")); w.Wdown=mk_mm(N("ffn_down.weight")); - ok &= w.attn_norm&&w.Wq&&w.Wk&&w.Wv&&w.Wo&&w.q_norm&&w.k_norm&&w.ffn_norm&&w.Wgate&&w.Wup&&w.Wdown; - } + WeightLoader L("gr00tn1d7", g, m->ctx_weights, m->matmul_type); - m->vlln_w=mk_f32("aex.vlln.weight"); m->vlln_b=mk_f32("aex.vlln.bias"); - m->vlsa.resize(m->vlsa_layers); - for (int64_t i = 0; i < m->vlsa_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "aex.vlsa.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vlsa[i]; - w.n1w=mk_f32(N("norm1.weight")); w.n1b=mk_f32(N("norm1.bias")); w.n3w=mk_f32(N("norm3.weight")); w.n3b=mk_f32(N("norm3.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.n1w&&w.n1b&&w.n3w&&w.n3b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } - m->se_l1W=mk_f32("aex.state_enc.l1.W"); m->se_l1b=mk_f32("aex.state_enc.l1.b"); m->se_l2W=mk_f32("aex.state_enc.l2.W"); m->se_l2b=mk_f32("aex.state_enc.l2.b"); - m->ae_W1W=mk_f32("aex.act_enc.W1.W"); m->ae_W1b=mk_f32("aex.act_enc.W1.b"); m->ae_W2W=mk_f32("aex.act_enc.W2.W"); m->ae_W2b=mk_f32("aex.act_enc.W2.b"); m->ae_W3W=mk_f32("aex.act_enc.W3.W"); m->ae_W3b=mk_f32("aex.act_enc.W3.b"); - m->ad_l1W=mk_f32("aex.act_dec.l1.W"); m->ad_l1b=mk_f32("aex.act_dec.l1.b"); m->ad_l2W=mk_f32("aex.act_dec.l2.W"); m->ad_l2b=mk_f32("aex.act_dec.l2.b"); - m->pos_embd=mk_f32("aex.pos_embd"); - m->te_l1W=mk_mm("aex.dit.time_emb.l1.weight"); m->te_l1b=mk_f32("aex.dit.time_emb.l1.bias"); m->te_l2W=mk_mm("aex.dit.time_emb.l2.weight"); m->te_l2b=mk_f32("aex.dit.time_emb.l2.bias"); - m->dit.resize(m->dit_layers); - for (int64_t i = 0; i < m->dit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "aex.dit.%lld.%s", (long long) i, s); return p; }; - auto & w = m->dit[i]; - w.adaln_w=mk_mm(N("adaln.weight")); w.adaln_b=mk_f32(N("adaln.bias")); - if (fuse) { - const std::string pre = "aex.dit." + std::to_string((long long) i) + "."; - const std::string qn=pre+"attn_q.weight", kn=pre+"attn_k.weight", vn=pre+"attn_v.weight"; - const std::string qb=pre+"attn_q.bias", kb=pre+"attn_k.bias", vb=pre+"attn_v.bias"; - if (m->dit_interleave && (i % 2 == 1)) { - const std::string ow=pre+"attn_qkv.fused.w", ob=pre+"attn_qkv.fused.b"; - w.Wqkv=mk_fused(ow.c_str(), {qn.c_str(),kn.c_str(),vn.c_str()}, m->matmul_type); - w.bqkv=mk_fused(ob.c_str(), {qb.c_str(),kb.c_str(),vb.c_str()}, GGML_TYPE_F32); - ok &= w.Wqkv&&w.bqkv; - } else { - const std::string ow=pre+"attn_kv.fused.w", ob=pre+"attn_kv.fused.b"; - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); - w.Wkv=mk_fused(ow.c_str(), {kn.c_str(),vn.c_str()}, m->matmul_type); - w.bkv=mk_fused(ob.c_str(), {kb.c_str(),vb.c_str()}, GGML_TYPE_F32); - ok &= w.Wq&&w.bq&&w.Wkv&&w.bkv; - } - } else { - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); - ok &= w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv; - } - w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.adaln_w&&w.adaln_b&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } - m->po1W=mk_mm("aex.dit.proj_out1.weight"); m->po1b=mk_f32("aex.dit.proj_out1.bias"); m->po2W=mk_mm("aex.dit.proj_out2.weight"); m->po2b=mk_f32("aex.dit.proj_out2.bias"); - ok &= m->vit_patch_w&&m->vit_patch_b&&m->vit_pos&&m->lm_output_norm&&m->vlln_w&&m->vlln_b&&m->se_l1W&&m->se_l1b&&m->se_l2W&&m->se_l2b&& - m->ae_W1W&&m->ae_W1b&&m->ae_W2W&&m->ae_W2b&&m->ae_W3W&&m->ae_W3b&&m->ad_l1W&&m->ad_l1b&&m->ad_l2W&&m->ad_l2b&&m->pos_embd&&m->te_l1W&&m->te_l1b&&m->te_l2W&&m->te_l2b&&m->po1W&&m->po1b&&m->po2W&&m->po2b; - if (!ok) { std::fprintf(stderr, "vla(gr00tn1d7): weight tensor setup failed\n"); return nullptr; } - - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(gr00tn1d7): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - std::set fused_dst; - for (const auto & fs : fused) fused_dst.insert(fs.dst); - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - if (fused_dst.count(t)) continue; - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(gr00tn1d7): failed to load %s (%zu vs %zu bytes)\n", ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } - for (const auto & fs : fused) { - std::vector buf; - for (const std::string & s : fs.srcs) { - std::vector b = g.read_convert(s.c_str(), fs.dst->type); - if (b.empty()) { std::fprintf(stderr, "vla(gr00tn1d7): fused fill: read %s failed\n", s.c_str()); return nullptr; } - buf.insert(buf.end(), b.begin(), b.end()); - } - if (buf.size() != ggml_nbytes(fs.dst)) { - std::fprintf(stderr, "vla(gr00tn1d7): fused fill: %s size %zu vs %zu\n", ggml_get_name(fs.dst), buf.size(), ggml_nbytes(fs.dst)); return nullptr; - } - ggml_backend_tensor_set(fs.dst, buf.data(), 0, buf.size()); - } - if (fuse) std::printf("vla(gr00tn1d7): QKV-fused DiT (self Wqkv / cross Wkv) - %zu fused tensors\n", fused.size()); + m->vit.declare(L, "vit", m->vit_layers); + m->lm.declare(L, "vlm"); + + m->vlln_w = L.f32("aex.vlln.weight"); + m->vlln_b = L.f32("aex.vlln.bias"); + m->vlsa.declare(L, "aex.vlsa", m->vlsa_layers, EncNames{"norm1", "norm3", "ff0", "ff2"}); + + m->aex.declare(L, "aex"); + m->dit.declare(L, "aex.dit", true, m->dit_interleave != 0); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + + std::printf("vla(gr00tn1d7): QKV-fused DiT (self Wqkv / cross Wkv)\n"); std::printf("vla(gr00tn1d7): weights resident in %.2f GiB (%s) - incl. Qwen3-VL vision tower + deepstack + vl_self_attention; embodiment id %lld\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", (long long) m->embodiment_id); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), + dtype_name(m->matmul_type), (long long) m->aex.embodiment_id); if (!m->build_caches()) { std::fprintf(stderr, "vla(gr00tn1d7): build_caches failed\n"); return nullptr; } return m; } @@ -532,18 +356,18 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * t_pos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_hidden, n_patches); ggml_set_input(t_pos); ggml_tensor * t_cos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, hd_vit, n_patches); ggml_set_input(t_cos); ggml_tensor * t_sin = ggml_new_tensor_2d(VC, GGML_TYPE_F32, hd_vit, n_patches); ggml_set_input(t_sin); - ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit_patch_w, t_patches), vit_patch_b), t_pos); + ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit.patch_w, t_patches), vit.patch_b), t_pos); ggml_set_output(h); ggml_tensor * stash[3] = {nullptr, nullptr, nullptr}; for (int64_t i = 0; i < vit_layers; ++i) { - h = build_vit_layer(VC, vit[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); + h = build_vit_layer(VC, vit.blk[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); ggml_set_output(h); for (int j = 0; j < 3; ++j) if (i == deepstack_idx[j]) stash[j] = h; } ggml_tensor * ds_out[3]; - for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } - ggml_tensor * vit_embeds = build_merger(VC, merger, h, vit_hidden, m2, connector_ln_eps, true); + for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } + ggml_tensor * vit_embeds = build_merger(VC, vit.merger, h, vit_hidden, m2, connector_ln_eps, true); ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); @@ -653,7 +477,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * h = t_embeds; for (int64_t i = 0; i < lm_layers; ++i) { - h = build_qwen3_layer(C, *this, lm[i], h, t_pos, t_lmmask, SEQ); + h = lm.block(C, lm.blk[i], h, t_pos, t_lmmask, SEQ); if (inject_deepstack && i < 3) h = ggml_add(C, h, t_ds[i]); if (do_dump) { ggml_set_output(h); lm_h_dump.push_back(h); } } @@ -664,7 +488,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { vl_embs = ggml_add(C, ggml_mul(C, ggml_norm(C, eagle, vlln_eps), vlln_w), vlln_b); if (do_dump) { ggml_set_output(vl_embs); vlsa_dump.push_back(vl_embs); } for (int64_t i = 0; i < vlsa_layers; ++i) { - vl_embs = build_vlsa_layer(C, vlsa[i], vl_embs, SEQ, vlsa_heads, vlsa_head_dim, bb_embed_dim, vlsa_ln_eps); + vl_embs = vlsa.block(C, vlsa.blk[i], vl_embs, SEQ); if (do_dump) { ggml_set_output(vl_embs); vlsa_dump.push_back(vl_embs); } } ggml_set_name(vl_embs, "vl_embs"); ggml_set_output(vl_embs); @@ -672,7 +496,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * vl_img = ggml_get_rows(C, vl_embs, t_img_idx); ggml_tensor * vl_txt = (t_txt_idx ? ggml_get_rows(C, vl_embs, t_txt_idx) : vl_img); - ggml_tensor * state_features = cat_linear(C, se_l2W, se_l2b, embodiment_id, ggml_relu(C, cat_linear(C, se_l1W, se_l1b, embodiment_id, t_state))); + ggml_tensor * state_features = cat_linear(C, aex.se_l2W, aex.se_l2b, aex.embodiment_id, ggml_relu(C, cat_linear(C, aex.se_l1W, aex.se_l1b, aex.embodiment_id, t_state))); const float dt = 1.0f / (float) num_steps; const int64_t every2 = 2 * attend_text_every_n; @@ -681,15 +505,15 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < dit_layers; ++i) { if (dit_interleave && (i % 2 == 1)) continue; ggml_tensor * enc = (i % every2 == 0) ? vl_txt : vl_img; - dit_kv(C, *this, dit[i], enc, &Kc[i], &Vc[i]); + dit.kv(C, dit.blk[i], enc, &Kc[i], &Vc[i]); } ggml_tensor * actions = t_x0; for (int64_t s = 0; s < num_steps; ++s) { - ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, te_l1W, t_tproj[s]), te_l1b))), te_l2b); - ggml_tensor * a_emb = cat_linear(C, ae_W1W, ae_W1b, embodiment_id, actions); - ggml_tensor * x_w2 = ggml_silu(C, cat_linear(C, ae_W2W, ae_W2b, embodiment_id, ggml_concat(C, a_emb, t_tau[s], 0))); - ggml_tensor * af = ggml_add(C, cat_linear(C, ae_W3W, ae_W3b, embodiment_id, x_w2), ggml_view_2d(C, pos_embd, E, AH, pos_embd->nb[1], 0)); + ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, dit.te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, dit.te_l1W, t_tproj[s]), dit.te_l1b))), dit.te_l2b); + ggml_tensor * a_emb = cat_linear(C, aex.ae_W1W, aex.ae_W1b, aex.embodiment_id, actions); + ggml_tensor * x_w2 = ggml_silu(C, cat_linear(C, aex.ae_W2W, aex.ae_W2b, aex.embodiment_id, ggml_concat(C, a_emb, t_tau[s], 0))); + ggml_tensor * af = ggml_add(C, cat_linear(C, aex.ae_W3W, aex.ae_W3b, aex.embodiment_id, x_w2), ggml_view_2d(C, aex.pos_embd, E, AH, aex.pos_embd->nb[1], 0)); ggml_tensor * sa = ggml_concat(C, state_features, af, 1); ggml_tensor * hh = sa; for (int64_t i = 0; i < dit_layers; ++i) { @@ -697,14 +521,14 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { if (dit_interleave && (i % 2 == 1)) enc = nullptr; else if (i % every2 == 0) enc = vl_txt; else enc = vl_img; - hh = build_dit_block(C, *this, dit[i], hh, temb, enc, Kc[i], Vc[i]); + hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); } - ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, po1W, ggml_silu(C, temb)), po1b); + ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, dit.po1W, ggml_silu(C, temb)), dit.po1b); ggml_tensor * sh = ggml_view_1d(C, po, dit_hidden, 0), * sc = ggml_view_1d(C, po, dit_hidden, (size_t) dit_hidden * sizeof(float)); ggml_tensor * hn = ggml_norm(C, hh, norm_out_eps); ggml_tensor * h_mod = ggml_add(C, ggml_add(C, hn, ggml_mul(C, hn, sc)), sh); - ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, po2W, h_mod), po2b); - ggml_tensor * pred = cat_linear(C, ad_l2W, ad_l2b, embodiment_id, ggml_relu(C, cat_linear(C, ad_l1W, ad_l1b, embodiment_id, model_output))); + ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, dit.po2W, h_mod), dit.po2b); + ggml_tensor * pred = cat_linear(C, aex.ad_l2W, aex.ad_l2b, aex.embodiment_id, ggml_relu(C, cat_linear(C, aex.ad_l1W, aex.ad_l1b, aex.embodiment_id, model_output))); ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t) (Nsa - AH) * pred->nb[1])); actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); } diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 188090c..dcce3b2 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -23,7 +23,11 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" +#include "layers/linear.h" +#include "layers/norm.h" +#include "modules/dit_head.h" +#include "modules/qwen3_lm.h" #include "modules/qwen3vl_vit.h" #include "env_flag.h" @@ -44,9 +48,6 @@ namespace vla { namespace { -struct Qwen3LayerW { ggml_tensor *attn_norm,*Wq,*Wk,*Wv,*Wo,*q_norm,*k_norm,*ffn_norm,*Wgate,*Wup,*Wdown; }; -struct DitLayerW { ggml_tensor *adaln_w,*adaln_b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; }; - } struct VlaJepaModelArch : public ModelArchBase { @@ -92,20 +93,14 @@ struct VlaJepaModelArch : public ModelArchBase { float vit_ln_eps=1e-6f, vit_rope_base=10000.0f, lm_rms_eps=1e-6f, lm_rope_base=5000000.0f, connector_ln_eps=1e-6f; float dit_ln_eps=1e-5f, dit_norm_out_eps=1e-6f; - ggml_tensor *vit_patch_w=nullptr,*vit_patch_b=nullptr,*vit_pos=nullptr; - std::vector vit; - MergerW deepstack[3]; - MergerW merger; - ggml_tensor *lm_output_norm=nullptr; - std::vector lm; + Qwen3VLTower vit; + Qwen3LM lm; + DitHead dit; ggml_tensor *ae_l1W=nullptr,*ae_l1b=nullptr,*ae_l2W=nullptr,*ae_l2b=nullptr,*ae_l3W=nullptr,*ae_l3b=nullptr; ggml_tensor *se_l1W=nullptr,*se_l1b=nullptr,*se_l2W=nullptr,*se_l2b=nullptr; ggml_tensor *ad_l1W=nullptr,*ad_l1b=nullptr,*ad_l2W=nullptr,*ad_l2b=nullptr; ggml_tensor *future_tokens=nullptr,*pos_embd=nullptr; - ggml_tensor *te_l1W=nullptr,*te_l1b=nullptr,*te_l2W=nullptr,*te_l2b=nullptr; - std::vector dit; - ggml_tensor *po1W=nullptr,*po1b=nullptr,*po2W=nullptr,*po2b=nullptr; bool caches_ready = false; std::vector c_grow, c_gcol; @@ -120,56 +115,7 @@ struct VlaJepaModelArch : public ModelArchBase { namespace { -ggml_tensor * build_qwen3_layer(ggml_context * C, const VlaJepaModelArch & m, const Qwen3LayerW & w, - ggml_tensor * h, ggml_tensor * positions, ggml_tensor * mask, int64_t seq) { - const int64_t hd = m.lm_head_dim, n_q = m.n_q, n_kv = m.n_kv, hq = n_q * hd; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * hn = ggml_mul(C, ggml_rms_norm(C, h, m.lm_rms_eps), w.attn_norm); - ggml_tensor * qp = ggml_mul_mat(C, w.Wq, hn); - ggml_tensor * kp = ggml_mul_mat(C, w.Wk, hn); - ggml_tensor * vp = ggml_mul_mat(C, w.Wv, hn); - ggml_tensor * qh = ggml_reshape_3d(C, qp, hd, n_q, seq); - ggml_tensor * kh = ggml_reshape_3d(C, kp, hd, n_kv, seq); - ggml_tensor * vh = ggml_reshape_3d(C, vp, hd, n_kv, seq); - ggml_tensor * qn = ggml_mul(C, ggml_rms_norm(C, qh, m.lm_rms_eps), w.q_norm); - ggml_tensor * kn = ggml_mul(C, ggml_rms_norm(C, kh, m.lm_rms_eps), w.k_norm); - int sections[4] = { 24, 20, 20, 0 }; - ggml_tensor * qr = ggml_rope_multi(C, qn, positions, nullptr, (int) hd, sections, GGML_ROPE_TYPE_IMROPE, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * kr = ggml_rope_multi(C, kn, positions, nullptr, (int) hd, sections, GGML_ROPE_TYPE_IMROPE, 0, m.lm_rope_base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, qr, 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, kr, 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, vh, 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, mask, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hq, seq); - ggml_tensor * h_attn = ggml_add(C, h, ggml_mul_mat(C, w.Wo, att)); - ggml_tensor * hn2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); - ggml_tensor * gate = ggml_silu(C, ggml_mul_mat(C, w.Wgate, hn2)); - ggml_tensor * up = ggml_mul_mat(C, w.Wup, hn2); - return ggml_add(C, h_attn, ggml_mul_mat(C, w.Wdown, ggml_mul(C, gate, up))); -} -ggml_tensor * build_dit_block(ggml_context * C, const VlaJepaModelArch & m, const DitLayerW & w, - ggml_tensor * h, ggml_tensor * temb, ggml_tensor * enc) { - const int64_t hd = m.dit_head_dim, heads = m.dit_heads, dim = m.dit_hidden, Tk = h->ne[1]; - const float scale = 1.0f / std::sqrt((float) hd); - ggml_tensor * n = adaln(C, h, temb, w.adaln_w, w.adaln_b, dim, m.dit_ln_eps); - ggml_tensor * kv = enc ? enc : n; - const int64_t Tkv = kv->ne[1]; - ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n), w.bq); - ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, kv), w.bk); - ggml_tensor * v = ggml_add(C, ggml_mul_mat(C, w.Wv, kv), w.bv); - ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, heads, Tk), 0, 2, 1, 3)); - ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, heads, Tkv), 0, 2, 1, 3)); - ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, hd, heads, Tkv), 1, 2, 0, 3)); - ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - ggml_tensor * aw = ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); - ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), dim, Tk); - ggml_tensor * h1 = ggml_add(C, h, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); - ggml_tensor * n3 = ggml_norm(C, h1, m.dit_ln_eps); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wff2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wff0, n3), w.bff0))), w.bff2); - return ggml_add(C, h1, ff); -} bool load_config(const gguf_reader & g, VlaJepaModelArch & m, Config & cfg) { auto U = [&](const char * k, int64_t & dst) { if (g.has(k)) dst = (int64_t) g.u32(k); }; @@ -209,6 +155,29 @@ bool load_config(const gguf_reader & g, VlaJepaModelArch & m, Config & cfg) { return false; } + m.lm.cfg.hidden = m.lm_hidden; + m.lm.cfg.layers = m.lm_layers; + m.lm.cfg.n_q = m.n_q; + m.lm.cfg.n_kv = m.n_kv; + m.lm.cfg.head_dim = m.lm_head_dim; + m.lm.cfg.inter = m.lm_inter; + m.lm.cfg.rms_eps = m.lm_rms_eps; + m.lm.cfg.flash_attn = flash_attn_enabled(); + m.lm.cfg.rope.type = GGML_ROPE_TYPE_IMROPE; + m.lm.cfg.rope.n_dims = (int) m.lm_head_dim; + m.lm.cfg.rope.freq_base = m.lm_rope_base; + m.lm.cfg.rope.sections[0] = 24; + m.lm.cfg.rope.sections[1] = 20; + m.lm.cfg.rope.sections[2] = 20; + m.lm.cfg.rope.sections[3] = 0; + + m.dit.cfg.hidden = m.dit_hidden; + m.dit.cfg.heads = m.dit_heads; + m.dit.cfg.head_dim = m.dit_head_dim; + m.dit.cfg.layers = m.dit_layers; + m.dit.cfg.ln_eps = m.dit_ln_eps; + m.dit.cfg.norm_out_eps = m.dit_norm_out_eps; + cfg = Config{}; cfg.n_img = (m.image_target_size / m.patch_size / m.spatial_merge) * (m.image_target_size / m.patch_size / m.spatial_merge); cfg.n_lang = 1024; cfg.n_state = 1; @@ -261,89 +230,31 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, m->backend = b.handle; } - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(vla_jepa): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(vla_jepa): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; - - bool ok = true; - - m->vit_patch_w = mk_mm("vit.patch_embd.weight"); m->vit_patch_b = mk_f32("vit.patch_embd.bias"); m->vit_pos = mk_f32("vit.pos_embd"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wqkv=mk_mm(N("attn_qkv.weight")); w.bqkv=mk_f32(N("attn_qkv.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - ok &= w.ln1w&&w.ln1b&&w.ln2w&&w.ln2b&&w.Wqkv&&w.bqkv&&w.Wo&&w.bo&&w.Wfc1&&w.bfc1&&w.Wfc2&&w.bfc2; - } - for (int j = 0; j < 3; ++j) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.deepstack.%d.%s", j, s); return p; }; - auto & w = m->deepstack[j]; - w.nw=mk_f32(N("norm.weight")); w.nb=mk_f32(N("norm.bias")); w.fc1w=mk_mm(N("fc1.weight")); w.fc1b=mk_f32(N("fc1.bias")); w.fc2w=mk_mm(N("fc2.weight")); w.fc2b=mk_f32(N("fc2.bias")); - ok &= w.nw&&w.nb&&w.fc1w&&w.fc1b&&w.fc2w&&w.fc2b; - } - { auto & w = m->merger; - w.nw=mk_f32("vit.merger.norm.weight"); w.nb=mk_f32("vit.merger.norm.bias"); w.fc1w=mk_mm("vit.merger.fc1.weight"); w.fc1b=mk_f32("vit.merger.fc1.bias"); w.fc2w=mk_mm("vit.merger.fc2.weight"); w.fc2b=mk_f32("vit.merger.fc2.bias"); - ok &= w.nw&&w.nb&&w.fc1w&&w.fc1b&&w.fc2w&&w.fc2b; } - - m->lm_output_norm = mk_f32("vlm.output_norm.weight"); - m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vlm.blk.%lld.%s", (long long) i, s); return p; }; - auto & w = m->lm[i]; - w.attn_norm=mk_f32(N("attn_norm.weight")); - w.Wq=mk_mm(N("attn_q.weight")); w.Wk=mk_mm(N("attn_k.weight")); w.Wv=mk_mm(N("attn_v.weight")); w.Wo=mk_mm(N("attn_o.weight")); - w.q_norm=mk_f32(N("attn_q_norm.weight")); w.k_norm=mk_f32(N("attn_k_norm.weight")); w.ffn_norm=mk_f32(N("ffn_norm.weight")); - w.Wgate=mk_mm(N("ffn_gate.weight")); w.Wup=mk_mm(N("ffn_up.weight")); w.Wdown=mk_mm(N("ffn_down.weight")); - ok &= w.attn_norm&&w.Wq&&w.Wk&&w.Wv&&w.Wo&&w.q_norm&&w.k_norm&&w.ffn_norm&&w.Wgate&&w.Wup&&w.Wdown; - } - m->ae_l1W=mk_f32("ah.act_enc.l1.weight"); m->ae_l1b=mk_f32("ah.act_enc.l1.bias"); - m->ae_l2W=mk_f32("ah.act_enc.l2.weight"); m->ae_l2b=mk_f32("ah.act_enc.l2.bias"); - m->ae_l3W=mk_f32("ah.act_enc.l3.weight"); m->ae_l3b=mk_f32("ah.act_enc.l3.bias"); - m->se_l1W=mk_f32("ah.state_enc.l1.weight"); m->se_l1b=mk_f32("ah.state_enc.l1.bias"); - m->se_l2W=mk_f32("ah.state_enc.l2.weight"); m->se_l2b=mk_f32("ah.state_enc.l2.bias"); - m->ad_l1W=mk_f32("ah.act_dec.l1.weight"); m->ad_l1b=mk_f32("ah.act_dec.l1.bias"); - m->ad_l2W=mk_f32("ah.act_dec.l2.weight"); m->ad_l2b=mk_f32("ah.act_dec.l2.bias"); - m->future_tokens=mk_f32("ah.future_tokens"); m->pos_embd=mk_f32("ah.pos_embd"); - m->te_l1W=mk_mm("ah.time_emb.l1.weight"); m->te_l1b=mk_f32("ah.time_emb.l1.bias"); - m->te_l2W=mk_mm("ah.time_emb.l2.weight"); m->te_l2b=mk_f32("ah.time_emb.l2.bias"); - m->dit.resize(m->dit_layers); - for (int64_t i = 0; i < m->dit_layers && ok; ++i) { - char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "ah.dit.%lld.%s", (long long) i, s); return p; }; - auto & w = m->dit[i]; - w.adaln_w=mk_mm(N("adaln.weight")); w.adaln_b=mk_f32(N("adaln.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wff0=mk_mm(N("ff0.weight")); w.bff0=mk_f32(N("ff0.bias")); w.Wff2=mk_mm(N("ff2.weight")); w.bff2=mk_f32(N("ff2.bias")); - ok &= w.adaln_w&&w.adaln_b&&w.Wq&&w.bq&&w.Wk&&w.bk&&w.Wv&&w.bv&&w.Wo&&w.bo&&w.Wff0&&w.bff0&&w.Wff2&&w.bff2; - } - m->po1W=mk_mm("ah.proj_out1.weight"); m->po1b=mk_f32("ah.proj_out1.bias"); m->po2W=mk_mm("ah.proj_out2.weight"); m->po2b=mk_f32("ah.proj_out2.bias"); - ok &= m->vit_patch_w&&m->vit_patch_b&&m->vit_pos&&m->lm_output_norm&&m->ae_l1W&&m->ae_l2W&&m->ae_l3W&&m->se_l1W&&m->se_l2W&&m->ad_l1W&&m->ad_l2W&& - m->future_tokens&&m->pos_embd&&m->te_l1W&&m->te_l1b&&m->te_l2W&&m->te_l2b&&m->po1W&&m->po1b&&m->po2W&&m->po2b; - if (!ok) { std::fprintf(stderr, "vla(vla_jepa): weight tensor setup failed\n"); return nullptr; } - - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(vla_jepa): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(vla_jepa): failed to load %s (%zu vs %zu bytes)\n", ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } - std::printf("vla(vla_jepa): weights resident in %.2f GiB (%s) - Qwen3-VL backbone + deepstack + DiT-B head\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); + WeightLoader L("vla_jepa", g, m->ctx_weights, m->matmul_type); + + m->vit.declare(L, "vit", m->vit_layers); + m->lm.declare(L, "vlm"); + + m->ae_l1W = L.f32("ah.act_enc.l1.weight"); m->ae_l1b = L.f32("ah.act_enc.l1.bias"); + m->ae_l2W = L.f32("ah.act_enc.l2.weight"); m->ae_l2b = L.f32("ah.act_enc.l2.bias"); + m->ae_l3W = L.f32("ah.act_enc.l3.weight"); m->ae_l3b = L.f32("ah.act_enc.l3.bias"); + m->se_l1W = L.f32("ah.state_enc.l1.weight"); m->se_l1b = L.f32("ah.state_enc.l1.bias"); + m->se_l2W = L.f32("ah.state_enc.l2.weight"); m->se_l2b = L.f32("ah.state_enc.l2.bias"); + m->ad_l1W = L.f32("ah.act_dec.l1.weight"); m->ad_l1b = L.f32("ah.act_dec.l1.bias"); + m->ad_l2W = L.f32("ah.act_dec.l2.weight"); m->ad_l2b = L.f32("ah.act_dec.l2.bias"); + m->future_tokens = L.f32("ah.future_tokens"); + m->pos_embd = L.f32("ah.pos_embd"); + + m->dit.declare(L, "ah.dit", false, false, "ah"); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + + std::printf("vla(vla_jepa): weights resident in %.2f GiB (%s)\n", + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->matmul_type)); if (!m->build_caches()) { std::fprintf(stderr, "vla(vla_jepa): build_caches failed\n"); return nullptr; } return m; } @@ -437,17 +348,17 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * t_pos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_hidden, n_patches); ggml_set_input(t_pos); ggml_tensor * t_cos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, hd_vit, n_patches); ggml_set_input(t_cos); ggml_tensor * t_sin = ggml_new_tensor_2d(VC, GGML_TYPE_F32, hd_vit, n_patches); ggml_set_input(t_sin); - ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit_patch_w, t_patches), vit_patch_b), t_pos); + ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit.patch_w, t_patches), vit.patch_b), t_pos); ggml_set_output(h); ggml_tensor * stash[3] = {nullptr, nullptr, nullptr}; for (int64_t i = 0; i < vit_layers; ++i) { - h = build_vit_layer(VC, vit[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); + h = build_vit_layer(VC, vit.blk[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); ggml_set_output(h); for (int j = 0; j < 3; ++j) if (i == deepstack_idx[j]) stash[j] = h; } ggml_tensor * ds_out[3]; - for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } - ggml_tensor * vit_embeds = build_merger(VC, merger, h, vit_hidden, m2, connector_ln_eps, true); + for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } + ggml_tensor * vit_embeds = build_merger(VC, vit.merger, h, vit_hidden, m2, connector_ln_eps, true); ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); @@ -514,7 +425,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { for (int j = 0; j < 3; ++j) { t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_ds[j]); } ggml_tensor * hh = t_embeds; for (int64_t i = 0; i < lm_layers; ++i) { - hh = build_qwen3_layer(C, *this, lm[i], hh, t_pos2, t_lmmask, SEQ); + hh = lm.block(C, lm.blk[i], hh, t_pos2, t_lmmask, SEQ); if (i < 3) hh = ggml_add(C, hh, t_ds[i]); } ggml_tensor * eagle = hh; @@ -598,7 +509,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * actions = t_x0; for (int64_t s = 0; s < num_steps; ++s) { - ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, te_l1W, t_tproj[s]), te_l1b))), te_l2b); + ggml_tensor * temb = ggml_add(C, ggml_mul_mat(C, dit.te_l2W, ggml_silu(C, ggml_add(C, ggml_mul_mat(C, dit.te_l1W, t_tproj[s]), dit.te_l1b))), dit.te_l2b); ggml_tensor * a_emb = ggml_add(C, ggml_mul_mat(C, ae_l1W, actions), ae_l1b); ggml_tensor * cat = ggml_concat(C, a_emb, t_tau[s], 0); @@ -610,14 +521,14 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * x = seq; for (int64_t i = 0; i < dit_layers; ++i) { ggml_tensor * enc = (i % 2 == 0) ? t_cond : nullptr; - x = build_dit_block(C, *this, dit[i], x, temb, enc); + x = dit.block(C, dit.blk[i], x, temb, enc); } - ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, po1W, ggml_silu(C, temb)), po1b); + ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, dit.po1W, ggml_silu(C, temb)), dit.po1b); ggml_tensor * sh = ggml_view_1d(C, po, dit_hidden, 0), * sc = ggml_view_1d(C, po, dit_hidden, (size_t) dit_hidden * sizeof(float)); ggml_tensor * xn = ggml_norm(C, x, dit_norm_out_eps); ggml_tensor * h_mod = ggml_add(C, ggml_add(C, xn, ggml_mul(C, xn, sc)), sh); - ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, po2W, h_mod), po2b); + ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, dit.po2W, h_mod), dit.po2b); step_pred[s] = model_output; ggml_tensor * last = ggml_cont(C, ggml_view_2d(C, model_output, OUTD, AH, model_output->nb[1], (size_t) (Nseq - AH) * model_output->nb[1])); diff --git a/src/modules/dit_head.cpp b/src/modules/dit_head.cpp index 01f8405..48f6942 100644 --- a/src/modules/dit_head.cpp +++ b/src/modules/dit_head.cpp @@ -24,11 +24,13 @@ namespace vla { -void DitHead::declare(WeightLoader & L, const char * prefix, bool fuse_qkv, bool interleave) { - te_l1W = L.gemm("%s.time_emb.l1.weight", prefix); - te_l1b = L.f32 ("%s.time_emb.l1.bias", prefix); - te_l2W = L.gemm("%s.time_emb.l2.weight", prefix); - te_l2b = L.f32 ("%s.time_emb.l2.bias", prefix); +void DitHead::declare(WeightLoader & L, const char * prefix, bool fuse_qkv, bool interleave, const char * outer) { + if (!outer) outer = prefix; + + te_l1W = L.gemm("%s.time_emb.l1.weight", outer); + te_l1b = L.f32 ("%s.time_emb.l1.bias", outer); + te_l2W = L.gemm("%s.time_emb.l2.weight", outer); + te_l2b = L.f32 ("%s.time_emb.l2.bias", outer); blk.resize(cfg.layers); for (int64_t i=0; i blk; + MergerW deepstack[3]; + MergerW merger; + ggml_tensor * patch_w = nullptr; + ggml_tensor * patch_b = nullptr; + ggml_tensor * pos = nullptr; + + void declare(WeightLoader & L, const char * prefix, int64_t layers) { + patch_w = L.gemm("%s.patch_embd.weight", prefix); + patch_b = L.f32 ("%s.patch_embd.bias", prefix); + pos = L.f32 ("%s.pos_embd", prefix); + + blk.resize(layers); + for (int64_t i=0; i Date: Thu, 13 Aug 2026 22:32:15 +0700 Subject: [PATCH 10/21] share the siglip tower and gemma stack between pi0 and pi0.5 --- src/models/pi0.cpp | 143 +++++++------------------------------ src/models/pi05.cpp | 141 ++++++++++-------------------------- src/modules/gemma_expert.h | 64 +++++++++++++++++ 3 files changed, 130 insertions(+), 218 deletions(-) create mode 100644 src/modules/gemma_expert.h diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 4ec1ba7..3ba095a 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -13,6 +13,8 @@ // limitations under the License. #include "arch.h" +#include "modules/gemma_expert.h" +#include "modules/siglip_vit.h" #include "options.h" #include "model.h" @@ -48,21 +50,6 @@ namespace vla { namespace { -struct GemmaLayerW { - ggml_tensor * ln_in = nullptr; - ggml_tensor * Wq = nullptr; - ggml_tensor * Wk = nullptr; - ggml_tensor * Wv = nullptr; - ggml_tensor * Wo = nullptr; - ggml_tensor * ln_post = nullptr; - ggml_tensor * Wgate = nullptr; - ggml_tensor * Wup = nullptr; - ggml_tensor * Wdown = nullptr; -}; - -// SigLIP-So400m vision block weights (PaliGemma tower, built in-tree like gr00tn1d5). -struct SigLipLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; - // The +1 RMSNorm fixup is a Gemma quirk that must only touch the language towers, // never the SigLIP LayerNorms (whose names would otherwise never match anyway). bool is_gemma_norm(const std::string & name) { @@ -111,15 +98,11 @@ struct Pi0ModelArch : public ModelArchBase { int64_t vit_hidden = 1152, vit_layers = 27, vit_heads = 16; int64_t vit_image_size = 224, vit_patch_size = 14, vit_n_tokens = 256; float vit_ln_eps = 1e-6f; - ggml_tensor * vit_patch_w = nullptr, * vit_patch_b = nullptr, * vit_pos = nullptr; - ggml_tensor * vit_post_ln_w = nullptr, * vit_post_ln_b = nullptr; - std::vector vit; + SigLipTower vit; ggml_tensor * mm_proj_w = nullptr, * mm_proj_b = nullptr; - std::vector pl_layers; - - std::vector ex_layers; - ggml_tensor * ex_final_norm = nullptr; + GemmaStack pl; + GemmaStack ex; ggml_tensor * W_sp = nullptr, * b_sp = nullptr; ggml_tensor * W_ain = nullptr, * b_ain = nullptr; @@ -146,7 +129,7 @@ namespace { // cost. (The evo1 SR drop this used to cite did not reproduce.) // VLA_PI0_BF16_ACT is the better lever here: 9.1%, and its SR was measured. -ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, +ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps, ggml_type at) { const float scale = 1.0f / std::sqrt((float) head_dim); @@ -420,97 +403,25 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(pi0): ggml_init(ctx_weights) failed\n"); return nullptr; } } - ggml_context * W = m->ctx_weights; - std::vector weights; - // A miss returns before pushing, so the null scan below cannot see it. - bool missing = false; - - auto mk = [&](const char * name, ggml_type type, int n_dims, const int64_t * ne) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi0): missing tensor %s\n", name); missing = true; return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), n_dims, ne); - ggml_set_name(t, name); - weights.push_back(t); - return t; - }; + WeightLoader L("pi0", g, m->ctx_weights, m->matmul_type); - auto mk_mm = [&](const char * name) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi0): missing tensor %s\n", name); missing = true; return nullptr; } - return mk(name, m->matmul_type, GGML_MAX_DIMS, gt->ne); - }; - auto mk_f32 = [&](const char * name) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi0): missing tensor %s\n", name); missing = true; return nullptr; } - return mk(name, GGML_TYPE_F32, GGML_MAX_DIMS, gt->ne); - }; + m->vit.declare(L, "vit", m->vit_layers); + m->mm_proj_w = L.gemm ("mm.proj.weight"); + m->mm_proj_b = L.opt_f32("mm.proj.bias"); - auto load_layer = [&](const char * tower, int i, GemmaLayerW & lw) -> bool { - char b[256]; - auto suf = [&](const char * s) { std::snprintf(b, sizeof(b), "%s.blk.%d.%s", tower, i, s); return b; }; - lw.ln_in = mk_f32(suf("attn_norm.weight")); - lw.Wq = mk_mm (suf("attn_q.weight")); - lw.Wk = mk_mm (suf("attn_k.weight")); - lw.Wv = mk_mm (suf("attn_v.weight")); - lw.Wo = mk_mm (suf("attn_o.weight")); - lw.ln_post = mk_f32(suf("ffn_norm.weight")); - lw.Wgate = mk_mm (suf("ffn_gate.weight")); - lw.Wup = mk_mm (suf("ffn_up.weight")); - lw.Wdown = mk_mm (suf("ffn_down.weight")); - return lw.ln_in && lw.Wq && lw.Wk && lw.Wv && lw.Wo && lw.ln_post && lw.Wgate && lw.Wup && lw.Wdown; - }; + m->pl.declare(L, "vlm", cfg.n_layers, false); + m->ex.declare(L, "aex", cfg.n_layers, true); - // Vision tower weights (SigLIP-So400m + PaliGemma projector), bundled in the ckpt GGUF. - m->vit_patch_w = mk_f32("vit.patch_embd.weight"); - m->vit_patch_b = mk_f32("vit.patch_embd.bias"); - m->vit_pos = mk_f32("vit.pos_embd"); - m->vit_post_ln_w = mk_f32("vit.post_ln.weight"); - m->vit_post_ln_b = mk_f32("vit.post_ln.bias"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers; ++i) { - char p[64]; - auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return (const char *) p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - } - m->mm_proj_w = mk_mm("mm.proj.weight"); - m->mm_proj_b = g.meta("mm.proj.bias") ? mk_f32("mm.proj.bias") : nullptr; // PaliGemma projector bias (optional) - - m->pl_layers.resize(cfg.n_layers); - m->ex_layers.resize(cfg.n_layers); - for (int64_t i = 0; i < cfg.n_layers; ++i) { - if (!load_layer("vlm", (int) i, m->pl_layers[i])) return nullptr; - if (!load_layer("aex", (int) i, m->ex_layers[i])) return nullptr; - } - m->ex_final_norm = mk_f32("aex.output_norm.weight"); - m->W_sp = mk_f32("state_proj.weight"); m->b_sp = mk_f32("state_proj.bias"); - m->W_ain = mk_f32("action_in_proj.weight"); m->b_ain = mk_f32("action_in_proj.bias"); - m->W_at1 = mk_f32("action_time_mlp_in.weight"); m->b_at1 = mk_f32("action_time_mlp_in.bias"); - m->W_at2 = mk_f32("action_time_mlp_out.weight"); m->b_at2 = mk_f32("action_time_mlp_out.bias"); - m->W_aout = mk_f32("action_out_proj.weight"); m->b_aout = mk_f32("action_out_proj.bias"); - if (missing) { std::fprintf(stderr, "vla(pi0): checkpoint is missing weights\n"); return nullptr; } - for (ggml_tensor * t : weights) if (!t) { std::fprintf(stderr, "vla(pi0): weight tensor creation failed\n"); return nullptr; } - if (!m->ex_final_norm || !m->W_sp || !m->b_sp || !m->W_ain || !m->b_ain || - !m->W_at1 || !m->b_at1 || !m->W_at2 || !m->b_at2 || !m->W_aout || !m->b_aout) { - std::fprintf(stderr, "vla(pi0): failed to wire projection / norm tensors\n"); return nullptr; - } + m->W_sp = L.f32("state_proj.weight"); m->b_sp = L.f32("state_proj.bias"); + m->W_ain = L.f32("action_in_proj.weight"); m->b_ain = L.f32("action_in_proj.bias"); + m->W_at1 = L.f32("action_time_mlp_in.weight"); m->b_at1 = L.f32("action_time_mlp_in.bias"); + m->W_at2 = L.f32("action_time_mlp_out.weight"); m->b_at2 = L.f32("action_time_mlp_out.bias"); + m->W_aout = L.f32("action_out_proj.weight"); m->b_aout = L.f32("action_out_proj.bias"); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(pi0): ggml_backend_alloc_ctx_tensors failed (out of memory?)\n"); return nullptr; } - for (ggml_tensor * t : weights) { - std::vector bytes = g.read_convert(t->name, t->type, is_gemma_norm(t->name)); - if (bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(pi0): upload size mismatch for %s (%zu vs %zu)\n", - t->name, bytes.size(), ggml_nbytes(t)); - return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } std::printf("vla(pi0): resident weights = %.2f GiB\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0)); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0)); if (!load_stats(g, *m)) return nullptr; std::printf("vla(pi0): model loaded (n_threads=%d)\n", m->n_threads); @@ -552,13 +463,13 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { ggml_context * VC = vision_scratch.reset((size_t) 128 * 1024 * 1024); if (!VC) { std::fprintf(stderr, "vla(pi0): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, vit_image_size, vit_image_size, 3); ggml_set_input(t_px); - ggml_tensor * conv = ggml_conv_2d(VC, vit_patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); + ggml_tensor * conv = ggml_conv_2d(VC, vit.patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); // patch embed (conv_2d) stays F32; the tower runs in the activation dtype - ggml_tensor * h = as_type(VC, ggml_add(VC, ggml_add(VC, patches, vit_patch_b), vit_pos), act_type); + ggml_tensor * h = as_type(VC, ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos), act_type); for (int64_t i = 0; i < vit_layers; ++i) - h = build_siglip_layer(VC, vit[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps, act_type); - h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit_post_ln_w), vit_post_ln_b); + h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps, act_type); + h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = mm_act(VC, mm_proj_w, h, act_type); if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); @@ -626,7 +537,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { { ggml_tensor * h = prefix_embs; for (int64_t i = 0; i < n_layers; ++i) { - h = build_gemma_layer(C, pl_layers[i], h, t_prefix_pos, cfg, n_prefix, rope_base, + h = build_gemma_layer(C, pl.blk[i], h, t_prefix_pos, cfg, n_prefix, rope_base, nullptr, nullptr, nullptr, &cK[i], &cV[i], act_type); } @@ -640,11 +551,11 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { for (int step = 0; step < num_steps; ++step) { ggml_tensor * h = build_embed_suffix(C, *this, t_state, x_t, t_time[step]); for (int64_t i = 0; i < n_layers; ++i) { - h = build_gemma_layer(C, ex_layers[i], h, t_suffix_pos, cfg, n_suf, rope_base, + h = build_gemma_layer(C, ex.blk[i], h, t_suffix_pos, cfg, n_suf, rope_base, cK[i], cV[i], t_full_mask, nullptr, nullptr, act_type); } - ggml_tensor * h_final = ggml_mul(C, ggml_rms_norm(C, h, cfg.rms_eps), ex_final_norm); + ggml_tensor * h_final = ggml_mul(C, ggml_rms_norm(C, h, cfg.rms_eps), ex.output_norm); // row stride follows h_final's dtype, which is BF16 on the BF16 path const size_t rb = (size_t) hidden_ex * ggml_element_size(h_final); ggml_tensor * h_actions = ggml_view_2d(C, h_final, hidden_ex, chunk, rb, rb); diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 54acd0e..3b9dda5 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -13,6 +13,8 @@ // limitations under the License. #include "arch.h" +#include "modules/gemma_expert.h" +#include "modules/siglip_vit.h" #include "options.h" #include "model.h" @@ -46,18 +48,6 @@ namespace vla { namespace { -struct VlmLayerW { - ggml_tensor * ln_in = nullptr; - ggml_tensor * Wq = nullptr; - ggml_tensor * Wk = nullptr; - ggml_tensor * Wv = nullptr; - ggml_tensor * Wo = nullptr; - ggml_tensor * ln_post = nullptr; - ggml_tensor * Wgate = nullptr; - ggml_tensor * Wup = nullptr; - ggml_tensor * Wdown = nullptr; -}; - struct ExpertLayerW { ggml_tensor * ada_in_w = nullptr; ggml_tensor * ada_in_b = nullptr; @@ -72,9 +62,6 @@ struct ExpertLayerW { ggml_tensor * Wdown = nullptr; }; -// SigLIP-So400m vision block weights (PaliGemma tower, built in-tree like gr00tn1d5). -struct SigLipLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; - bool ends_with(const std::string & s, const char * sfx) { const size_t n = std::strlen(sfx); return s.size() >= n && s.compare(s.size() - n, n, sfx) == 0; @@ -121,12 +108,10 @@ struct Pi05ModelArch : public ModelArchBase { int64_t vit_hidden = 1152, vit_layers = 27, vit_heads = 16; int64_t vit_image_size = 224, vit_patch_size = 14, vit_n_tokens = 256; float vit_ln_eps = 1e-6f; - ggml_tensor * vit_patch_w = nullptr, * vit_patch_b = nullptr, * vit_pos = nullptr; - ggml_tensor * vit_post_ln_w = nullptr, * vit_post_ln_b = nullptr; - std::vector vit; + SigLipTower vit; ggml_tensor * mm_proj_w = nullptr, * mm_proj_b = nullptr; - std::vector pl_layers; + GemmaStack pl; std::vector ex_layers; ggml_tensor * ex_final_w = nullptr; @@ -150,7 +135,7 @@ namespace { // One pre-norm SigLIP encoder block, identical to gr00tn1d5's in-tree tower // (the PaliGemma vision tower is the same SigLIP-So400m/14). Bidirectional // attention (nullptr mask), F32 score accumulation, tanh GELU FFN. -ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, +ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { const float scale = 1.0f / std::sqrt((float) head_dim); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); @@ -172,7 +157,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_ // CHW-planar float image in [-1,1] for ggml_conv_2d (SigLIP mean/std 0.5). ggml_tensor * build_vlm_layer( - ggml_context * ctx, const VlmLayerW & w, + ggml_context * ctx, const GemmaLayerW & w, ggml_tensor * x_in, ggml_tensor * positions, const Config & cfg, int64_t seq, float rope_base, ggml_tensor ** k_out, ggml_tensor ** v_out) { @@ -475,89 +460,41 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, return mk(name, GGML_TYPE_F32, GGML_MAX_DIMS, gt->ne); }; - auto load_vlm = [&](int i, VlmLayerW & lw) -> bool { - char b[256]; - auto suf = [&](const char * s) { std::snprintf(b, sizeof(b), "vlm.blk.%d.%s", i, s); return b; }; - lw.ln_in = mk_f32(suf("attn_norm.weight")); - lw.Wq = mk_mm (suf("attn_q.weight")); - lw.Wk = mk_mm (suf("attn_k.weight")); - lw.Wv = mk_mm (suf("attn_v.weight")); - lw.Wo = mk_mm (suf("attn_o.weight")); - lw.ln_post = mk_f32(suf("ffn_norm.weight")); - lw.Wgate = mk_mm (suf("ffn_gate.weight")); - lw.Wup = mk_mm (suf("ffn_up.weight")); - lw.Wdown = mk_mm (suf("ffn_down.weight")); - return lw.ln_in && lw.Wq && lw.Wk && lw.Wv && lw.Wo && lw.ln_post && lw.Wgate && lw.Wup && lw.Wdown; - }; - auto load_expert = [&](int i, ExpertLayerW & lw) -> bool { - char b[256]; - auto suf = [&](const char * s) { std::snprintf(b, sizeof(b), "aex.blk.%d.%s", i, s); return b; }; - lw.ada_in_w = mk_f32(suf("attn_norm.weight")); - lw.ada_in_b = mk_f32(suf("attn_norm.bias")); - lw.Wq = mk_mm (suf("attn_q.weight")); - lw.Wk = mk_mm (suf("attn_k.weight")); - lw.Wv = mk_mm (suf("attn_v.weight")); - lw.Wo = mk_mm (suf("attn_o.weight")); - lw.ada_post_w = mk_f32(suf("ffn_norm.weight")); - lw.ada_post_b = mk_f32(suf("ffn_norm.bias")); - lw.Wgate = mk_mm (suf("ffn_gate.weight")); - lw.Wup = mk_mm (suf("ffn_up.weight")); - lw.Wdown = mk_mm (suf("ffn_down.weight")); - return lw.ada_in_w && lw.ada_in_b && lw.Wq && lw.Wk && lw.Wv && lw.Wo && - lw.ada_post_w && lw.ada_post_b && lw.Wgate && lw.Wup && lw.Wdown; - }; + WeightLoader L("pi05", g, m->ctx_weights, m->matmul_type); - // Vision tower weights (SigLIP-So400m + PaliGemma projector), bundled in the ckpt GGUF. - m->vit_patch_w = mk_f32("vit.patch_embd.weight"); - m->vit_patch_b = mk_f32("vit.patch_embd.bias"); - m->vit_pos = mk_f32("vit.pos_embd"); - m->vit_post_ln_w = mk_f32("vit.post_ln.weight"); - m->vit_post_ln_b = mk_f32("vit.post_ln.bias"); - m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers; ++i) { - char p[64]; - auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return (const char *) p; }; - auto & w = m->vit[i]; - w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); w.ln2w=mk_f32(N("ln2.weight")); w.ln2b=mk_f32(N("ln2.bias")); - w.Wq=mk_mm(N("attn_q.weight")); w.bq=mk_f32(N("attn_q.bias")); w.Wk=mk_mm(N("attn_k.weight")); w.bk=mk_f32(N("attn_k.bias")); - w.Wv=mk_mm(N("attn_v.weight")); w.bv=mk_f32(N("attn_v.bias")); w.Wo=mk_mm(N("attn_o.weight")); w.bo=mk_f32(N("attn_o.bias")); - w.Wfc1=mk_mm(N("fc1.weight")); w.bfc1=mk_f32(N("fc1.bias")); w.Wfc2=mk_mm(N("fc2.weight")); w.bfc2=mk_f32(N("fc2.bias")); - } - m->mm_proj_w = mk_mm("mm.proj.weight"); - m->mm_proj_b = g.meta("mm.proj.bias") ? mk_f32("mm.proj.bias") : nullptr; // PaliGemma projector bias (optional) + m->vit.declare(L, "vit", m->vit_layers); + m->mm_proj_w = L.gemm ("mm.proj.weight"); + m->mm_proj_b = L.opt_f32("mm.proj.bias"); + + m->pl.declare(L, "vlm", cfg.n_layers, false); - m->pl_layers.resize(cfg.n_layers); m->ex_layers.resize(cfg.n_layers); for (int64_t i = 0; i < cfg.n_layers; ++i) { - if (!load_vlm ((int) i, m->pl_layers[i])) return nullptr; - if (!load_expert((int) i, m->ex_layers[i])) return nullptr; - } - m->ex_final_w = mk_f32("aex.output_norm.weight"); - m->ex_final_b = mk_f32("aex.output_norm.bias"); - m->W_ain = mk_f32("action_in_proj.weight"); m->b_ain = mk_f32("action_in_proj.bias"); - m->W_tin = mk_f32("time_mlp_in.weight"); m->b_tin = mk_f32("time_mlp_in.bias"); - m->W_tout = mk_f32("time_mlp_out.weight"); m->b_tout = mk_f32("time_mlp_out.bias"); - m->W_aout = mk_f32("action_out_proj.weight"); m->b_aout = mk_f32("action_out_proj.bias"); - if (missing) { std::fprintf(stderr, "vla(pi05): checkpoint is missing weights\n"); return nullptr; } - for (ggml_tensor * t : weights) if (!t) { std::fprintf(stderr, "vla(pi05): weight tensor creation failed\n"); return nullptr; } - if (!m->ex_final_w || !m->ex_final_b || !m->W_ain || !m->b_ain || !m->W_tin || !m->b_tin || - !m->W_tout || !m->b_tout || !m->W_aout || !m->b_aout) { - std::fprintf(stderr, "vla(pi05): failed to wire projection / norm tensors\n"); return nullptr; + ExpertLayerW & w = m->ex_layers[i]; + w.ada_in_w = L.f32 ("aex.blk.%lld.attn_norm.weight", (long long)i); + w.ada_in_b = L.f32 ("aex.blk.%lld.attn_norm.bias", (long long)i); + w.Wq = L.gemm("aex.blk.%lld.attn_q.weight", (long long)i); + w.Wk = L.gemm("aex.blk.%lld.attn_k.weight", (long long)i); + w.Wv = L.gemm("aex.blk.%lld.attn_v.weight", (long long)i); + w.Wo = L.gemm("aex.blk.%lld.attn_o.weight", (long long)i); + w.ada_post_w = L.f32 ("aex.blk.%lld.ffn_norm.weight", (long long)i); + w.ada_post_b = L.f32 ("aex.blk.%lld.ffn_norm.bias", (long long)i); + w.Wgate = L.gemm("aex.blk.%lld.ffn_gate.weight", (long long)i); + w.Wup = L.gemm("aex.blk.%lld.ffn_up.weight", (long long)i); + w.Wdown = L.gemm("aex.blk.%lld.ffn_down.weight", (long long)i); } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(pi05): ggml_backend_alloc_ctx_tensors failed (out of memory?)\n"); return nullptr; } - for (ggml_tensor * t : weights) { - std::vector bytes = g.read_convert(t->name, t->type, is_gemma_norm_pi05(t->name)); - if (bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(pi05): upload size mismatch for %s (%zu vs %zu)\n", - t->name, bytes.size(), ggml_nbytes(t)); - return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } + m->ex_final_w = L.f32("aex.output_norm.weight"); + m->ex_final_b = L.f32("aex.output_norm.bias"); + m->W_ain = L.f32("action_in_proj.weight"); m->b_ain = L.f32("action_in_proj.bias"); + m->W_tin = L.f32("time_mlp_in.weight"); m->b_tin = L.f32("time_mlp_in.bias"); + m->W_tout = L.f32("time_mlp_out.weight"); m->b_tout = L.f32("time_mlp_out.bias"); + m->W_aout = L.f32("action_out_proj.weight"); m->b_aout = L.f32("action_out_proj.bias"); + + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + std::printf("vla(pi05): resident weights = %.2f GiB\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0)); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0)); if (!load_stats(g, *m)) return nullptr; std::printf("vla(pi05): model loaded (n_threads=%d)\n", m->n_threads); @@ -598,12 +535,12 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { ggml_context * VC = vision_scratch.reset((size_t) 128 * 1024 * 1024); if (!VC) { std::fprintf(stderr, "vla(pi05): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, vit_image_size, vit_image_size, 3); ggml_set_input(t_px); - ggml_tensor * conv = ggml_conv_2d(VC, vit_patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); + ggml_tensor * conv = ggml_conv_2d(VC, vit.patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); - ggml_tensor * h = ggml_add(VC, ggml_add(VC, patches, vit_patch_b), vit_pos); + ggml_tensor * h = ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos); for (int64_t i = 0; i < vit_layers; ++i) - h = build_siglip_layer(VC, vit[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps); - h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit_post_ln_w), vit_post_ln_b); + h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps); + h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = ggml_mul_mat(VC, mm_proj_w, h); if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); @@ -673,7 +610,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { { ggml_tensor * h = prefix_embs; for (int64_t i = 0; i < n_layers; ++i) { - h = build_vlm_layer(C, pl_layers[i], h, t_prefix_pos, cfg, n_prefix, rope_base, + h = build_vlm_layer(C, pl.blk[i], h, t_prefix_pos, cfg, n_prefix, rope_base, &cK[i], &cV[i]); } (void) h; diff --git a/src/modules/gemma_expert.h b/src/modules/gemma_expert.h new file mode 100644 index 0000000..1faf840 --- /dev/null +++ b/src/modules/gemma_expert.h @@ -0,0 +1,64 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Gemma decoder stack. pi0, pi0.5 and SmolVLA each run two of these with a +// shared attention: a prefix tower over the image and language tokens, and an +// action expert over the state and noisy-action tokens. + +#pragma once + +#include "loader.h" + +#include "ggml.h" + +#include +#include + +namespace vla { + +struct GemmaLayerW { + ggml_tensor * ln_in = nullptr; + ggml_tensor * Wq = nullptr; + ggml_tensor * Wk = nullptr; + ggml_tensor * Wv = nullptr; + ggml_tensor * Wo = nullptr; + ggml_tensor * ln_post = nullptr; + ggml_tensor * Wgate = nullptr; + ggml_tensor * Wup = nullptr; + ggml_tensor * Wdown = nullptr; +}; + +struct GemmaStack { + std::vector blk; + ggml_tensor * output_norm = nullptr; + + void declare(WeightLoader & L, const char * prefix, int64_t layers, bool with_output_norm) { + blk.resize(layers); + for (int64_t i=0; i Date: Thu, 13 Aug 2026 22:35:51 +0700 Subject: [PATCH 11/21] give the dinov2+siglip dual tower a module that both archs declare through --- src/models/openvla_oft.cpp | 59 ++++++++++---------------------------- src/models/vla_adapter.cpp | 57 +++++++++--------------------------- src/modules/dual_tower.h | 49 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 87 deletions(-) diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index cab6e97..ec41336 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -118,9 +118,7 @@ struct OpenVlaOftModelArch : public ModelArchBase { float head_ln_eps=1e-5f; int64_t stop_id=2; - ggml_tensor *d_patch_w,*d_patch_b,*d_cls,*d_reg,*d_pos; std::vector dvit; - ggml_tensor *s_patch_w,*s_patch_b,*s_pos; std::vector svit; - ggml_tensor *pj_fc1w,*pj_fc1b,*pj_fc2w,*pj_fc2b,*pj_fc3w,*pj_fc3b; + DualTower vis; ggml_tensor *token_embd,*lm_out_norm; std::vector lm; ggml_tensor *pp_fc1w,*pp_fc1b,*pp_fc2w,*pp_fc2b; ggml_tensor *h_ln1w,*h_ln1b,*h_fc1w,*h_fc1b,*h_ln2w,*h_ln2b,*h_fc2w,*h_fc2b; std::vector hblk; @@ -180,34 +178,12 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path m->ctx_weights = ggml_init(wp); ggml_context * W = m->ctx_weights; bool ok = true; - auto mk=[&](const char*name, ggml_type ty)->ggml_tensor*{ const ggml_tensor*gt=g.meta(name); - if(!gt){ std::fprintf(stderr,"vla(openvla_oft): missing %s\n",name); ok=false; return nullptr; } - ggml_tensor*t=ggml_new_tensor(W,g.resident_type(gt,ty),ggml_n_dims(gt),gt->ne); ggml_set_name(t,name); return t; }; - auto mm=[&](const char*n){ return mk(n,m->mt); }; - auto f32=[&](const char*n){ return mk(n,GGML_TYPE_F32); }; - - m->d_patch_w=mk("vis.d.patch.weight",GGML_TYPE_F32); m->d_patch_b=f32("vis.d.patch.bias"); - m->d_cls=f32("vis.d.cls"); m->d_reg=f32("vis.d.reg"); m->d_pos=f32("vis.d.pos"); - m->dvit.resize(m->d_layers); - for(int i=0;id_layers;++i){ auto&w=m->dvit[i]; char b[64]; - auto N=[&](const char*s){ std::snprintf(b,sizeof(b),"vis.d.blk.%d.%s",i,s); return (const char*)b; }; - w.n1w=f32(N("ln1.weight")); w.n1b=f32(N("ln1.bias")); w.n2w=f32(N("ln2.weight")); w.n2b=f32(N("ln2.bias")); - w.ls1=f32(N("ls1")); w.ls2=f32(N("ls2")); w.Wqkv=mm(N("qkv.weight")); w.bqkv=f32(N("qkv.bias")); - w.Wproj=mm(N("proj.weight")); w.bproj=f32(N("proj.bias")); w.Wfc1=mm(N("fc1.weight")); w.bfc1=f32(N("fc1.bias")); - w.Wfc2=mm(N("fc2.weight")); w.bfc2=f32(N("fc2.bias")); } - - m->s_patch_w=mk("vis.s.patch.weight",GGML_TYPE_F32); m->s_patch_b=f32("vis.s.patch.bias"); m->s_pos=f32("vis.s.pos"); - m->svit.resize(m->s_layers); - for(int i=0;is_layers;++i){ auto&w=m->svit[i]; char b[64]; - auto N=[&](const char*s){ std::snprintf(b,sizeof(b),"vis.s.blk.%d.%s",i,s); return (const char*)b; }; - w.n1w=f32(N("ln1.weight")); w.n1b=f32(N("ln1.bias")); w.n2w=f32(N("ln2.weight")); w.n2b=f32(N("ln2.bias")); - w.ls1=nullptr; w.ls2=nullptr; w.Wqkv=mm(N("qkv.weight")); w.bqkv=f32(N("qkv.bias")); - w.Wproj=mm(N("proj.weight")); w.bproj=f32(N("proj.bias")); w.Wfc1=mm(N("fc1.weight")); w.bfc1=f32(N("fc1.bias")); - w.Wfc2=mm(N("fc2.weight")); w.bfc2=f32(N("fc2.bias")); } - - m->pj_fc1w=mm("vis.proj.fc1.weight"); m->pj_fc1b=f32("vis.proj.fc1.bias"); - m->pj_fc2w=mm("vis.proj.fc2.weight"); m->pj_fc2b=f32("vis.proj.fc2.bias"); - m->pj_fc3w=mm("vis.proj.fc3.weight"); m->pj_fc3b=f32("vis.proj.fc3.bias"); + WeightLoader L("openvla_oft", g, m->ctx_weights, m->mt); + auto mk = [&](const char * n, ggml_type ty) { return ty == GGML_TYPE_F32 ? L.f32("%s", n) : L.gemm("%s", n); }; + auto mm = [&](const char * n) { return L.gemm("%s", n); }; + auto f32 = [&](const char * n) { return L.f32("%s", n); }; + + m->vis.declare(L, m->d_layers, m->s_layers); m->token_embd=mm("token_embd.weight"); m->lm_out_norm=f32("lm.output_norm.weight"); m->lm.resize(m->lm_layers); @@ -230,15 +206,10 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path w.lnw=f32(N("ln.weight")); w.lnb=f32(N("ln.bias")); w.linw=mm(N("lin.weight")); w.linb=f32(N("lin.bias")); } if(!ok){ std::fprintf(stderr,"vla(openvla_oft): weight setup failed\n"); return nullptr; } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if(!m->weight_buf){ std::fprintf(stderr,"vla(openvla_oft): alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for(ggml_tensor*t=ggml_get_first_tensor(W); t; t=ggml_get_next_tensor(W,t)){ - std::vector bytes=g.read_convert(ggml_get_name(t),t->type); - if(bytes.empty()||bytes.size()!=ggml_nbytes(t)){ std::fprintf(stderr,"vla(openvla_oft): load %s (%zu vs %zu)\n",ggml_get_name(t),bytes.size(),ggml_nbytes(t)); return nullptr; } - ggml_backend_tensor_set(t,bytes.data(),0,bytes.size()); - } + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + std::printf("vla(openvla_oft): weights resident %.2f GiB (%s) - DINOv2+SigLIP towers + Llama-2-7B + MLPResNet L1 head\n", - ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), m->mt==GGML_TYPE_F32?"F32":"BF16"); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->mt)); m->cfg.n_suffix = m->chunk; m->cfg.max_action_dim = m->action_dim; m->cfg.real_action_dim = m->action_dim; m->cfg.real_state_dim = m->proprio_dim; @@ -277,14 +248,14 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { for(int v=0; v dbuf, sbuf; diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index dbadc8e..1595788 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -122,9 +122,7 @@ struct VlaAdapterModelArch : public ModelArchBase { float head_rope_base=1e4f, head_ln_eps=1e-5f; int64_t stop_id=2; - ggml_tensor *d_patch_w,*d_patch_b,*d_cls,*d_reg,*d_pos; std::vector dvit; - ggml_tensor *s_patch_w,*s_patch_b,*s_pos; std::vector svit; - ggml_tensor *pj_fc1w,*pj_fc1b,*pj_fc2w,*pj_fc2b,*pj_fc3w,*pj_fc3b; + DualTower vis; ggml_tensor *token_embd,*action_queries,*lm_out_norm; std::vector lm; ggml_tensor *h_ln1w,*h_ln1b,*h_fc1w,*h_fc1b,*h_ln2w,*h_ln2b,*h_fc2w,*h_fc2b; ggml_tensor *pp_fc1w,*pp_fc1b,*pp_fc2w,*pp_fc2b; std::vector hblk; @@ -210,36 +208,14 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path m->ctx_weights = ggml_init(wp); ggml_context * W = m->ctx_weights; bool ok = true; - auto mk=[&](const char*name, ggml_type ty)->ggml_tensor*{ const ggml_tensor*gt=g.meta(name); - if(!gt){ std::fprintf(stderr,"vla(vla_adapter): missing %s\n",name); ok=false; return nullptr; } - ggml_tensor*t=ggml_new_tensor(W,g.resident_type(gt,ty),ggml_n_dims(gt),gt->ne); ggml_set_name(t,name); return t; }; - auto mm=[&](const char*n){ return mk(n,m->mt); }; - auto f32=[&](const char*n){ return mk(n,GGML_TYPE_F32); }; + WeightLoader L("vla_adapter", g, m->ctx_weights, m->mt); + auto mk = [&](const char * n, ggml_type ty) { return ty == GGML_TYPE_F32 ? L.f32("%s", n) : L.gemm("%s", n); }; + auto mm = [&](const char * n) { return L.gemm("%s", n); }; + auto f32 = [&](const char * n) { return L.f32("%s", n); }; char nm[96]; auto P=[&](const char*fmt,int i){ std::snprintf(nm,sizeof(nm),fmt,i); return (const char*)nm; }; - m->d_patch_w=mk("vis.d.patch.weight",GGML_TYPE_F32); m->d_patch_b=f32("vis.d.patch.bias"); - m->d_cls=f32("vis.d.cls"); m->d_reg=f32("vis.d.reg"); m->d_pos=f32("vis.d.pos"); - m->dvit.resize(m->d_layers); - for(int i=0;id_layers;++i){ auto&w=m->dvit[i]; char b[64]; - auto N=[&](const char*s){ std::snprintf(b,sizeof(b),"vis.d.blk.%d.%s",i,s); return (const char*)b; }; - w.n1w=f32(N("ln1.weight")); w.n1b=f32(N("ln1.bias")); w.n2w=f32(N("ln2.weight")); w.n2b=f32(N("ln2.bias")); - w.ls1=f32(N("ls1")); w.ls2=f32(N("ls2")); w.Wqkv=mm(N("qkv.weight")); w.bqkv=f32(N("qkv.bias")); - w.Wproj=mm(N("proj.weight")); w.bproj=f32(N("proj.bias")); w.Wfc1=mm(N("fc1.weight")); w.bfc1=f32(N("fc1.bias")); - w.Wfc2=mm(N("fc2.weight")); w.bfc2=f32(N("fc2.bias")); } - - m->s_patch_w=mk("vis.s.patch.weight",GGML_TYPE_F32); m->s_patch_b=f32("vis.s.patch.bias"); m->s_pos=f32("vis.s.pos"); - m->svit.resize(m->s_layers); - for(int i=0;is_layers;++i){ auto&w=m->svit[i]; char b[64]; - auto N=[&](const char*s){ std::snprintf(b,sizeof(b),"vis.s.blk.%d.%s",i,s); return (const char*)b; }; - w.n1w=f32(N("ln1.weight")); w.n1b=f32(N("ln1.bias")); w.n2w=f32(N("ln2.weight")); w.n2b=f32(N("ln2.bias")); - w.ls1=nullptr; w.ls2=nullptr; w.Wqkv=mm(N("qkv.weight")); w.bqkv=f32(N("qkv.bias")); - w.Wproj=mm(N("proj.weight")); w.bproj=f32(N("proj.bias")); w.Wfc1=mm(N("fc1.weight")); w.bfc1=f32(N("fc1.bias")); - w.Wfc2=mm(N("fc2.weight")); w.bfc2=f32(N("fc2.bias")); } - - m->pj_fc1w=mm("vis.proj.fc1.weight"); m->pj_fc1b=f32("vis.proj.fc1.bias"); - m->pj_fc2w=mm("vis.proj.fc2.weight"); m->pj_fc2b=f32("vis.proj.fc2.bias"); - m->pj_fc3w=mm("vis.proj.fc3.weight"); m->pj_fc3b=f32("vis.proj.fc3.bias"); + m->vis.declare(L, m->d_layers, m->s_layers); m->token_embd=mm("token_embd.weight"); m->action_queries=mm("action_queries.weight"); m->lm_out_norm=f32("lm.output_norm.weight"); m->lm.resize(m->lm_layers); @@ -268,15 +244,10 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path (void)P; if(!ok){ std::fprintf(stderr,"vla(vla_adapter): weight setup failed\n"); return nullptr; } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if(!m->weight_buf){ std::fprintf(stderr,"vla(vla_adapter): alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for(ggml_tensor*t=ggml_get_first_tensor(W); t; t=ggml_get_next_tensor(W,t)){ - std::vector bytes=g.read_convert(ggml_get_name(t),t->type); - if(bytes.empty()||bytes.size()!=ggml_nbytes(t)){ std::fprintf(stderr,"vla(vla_adapter): load %s (%zu vs %zu)\n",ggml_get_name(t),bytes.size(),ggml_nbytes(t)); return nullptr; } - ggml_backend_tensor_set(t,bytes.data(),0,bytes.size()); - } + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + std::printf("vla(vla_adapter): weights resident %.2f GiB (%s) - DINOv2+SigLIP towers + Qwen2.5-0.5B + Bridge head\n", - ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), m->mt==GGML_TYPE_F32?"F32":"BF16"); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->mt)); m->cfg.n_suffix = m->chunk; m->cfg.max_action_dim = m->action_dim; m->cfg.real_action_dim = m->action_dim; m->cfg.real_state_dim = m->proprio_dim; @@ -319,14 +290,14 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { for(int v=0; v dbuf, sbuf; diff --git a/src/modules/dual_tower.h b/src/modules/dual_tower.h index 20bc40a..edfcf80 100644 --- a/src/modules/dual_tower.h +++ b/src/modules/dual_tower.h @@ -19,6 +19,7 @@ #pragma once #include "ggml.h" +#include "loader.h" #include "model.h" #include @@ -29,6 +30,54 @@ namespace vla { struct ViTLayerW { ggml_tensor *n1w,*n1b,*n2w,*n2b,*ls1,*ls2,*Wqkv,*bqkv,*Wproj,*bproj,*Wfc1,*bfc1,*Wfc2,*bfc2; }; +// DINOv2 carries CLS + 4 register tokens and LayerScale; SigLIP carries +// neither, so its ls1/ls2 stay null and vit_block is told to skip them. +struct DualTower { + ggml_tensor *d_patch_w=nullptr,*d_patch_b=nullptr,*d_cls=nullptr,*d_reg=nullptr,*d_pos=nullptr; + ggml_tensor *s_patch_w=nullptr,*s_patch_b=nullptr,*s_pos=nullptr; + std::vector dvit, svit; + ggml_tensor *pj_fc1w=nullptr,*pj_fc1b=nullptr,*pj_fc2w=nullptr,*pj_fc2b=nullptr,*pj_fc3w=nullptr,*pj_fc3b=nullptr; + + void declare(WeightLoader & L, int64_t d_layers, int64_t s_layers) { + auto blocks = [&](std::vector & v, const char * pre, int64_t n, bool layer_scale) { + v.resize(n); + for (int64_t i=0; i Date: Thu, 13 Aug 2026 22:37:56 +0700 Subject: [PATCH 12/21] route evo1 and bitvla weight loading through the shared loader --- src/loader.cpp | 8 ++++++++ src/loader.h | 4 ++++ src/models/bitvla.cpp | 28 +++++++--------------------- src/models/evo1.cpp | 33 ++++++++++----------------------- 4 files changed, 29 insertions(+), 44 deletions(-) diff --git a/src/loader.cpp b/src/loader.cpp index 3bb2b4a..f547119 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -74,6 +74,14 @@ VLA_DECLARE_FN(f32_gemma_norm, GGML_TYPE_F32, true, true) #undef VLA_DECLARE_FN +ggml_tensor * WeightLoader::typed(ggml_type want, const char * fmt, ...) { + va_list ap; + va_start(ap, fmt); + ggml_tensor * t = declare(want, true, false, fmt, ap); + va_end(ap); + return t; +} + ggml_tensor * WeightLoader::fuse_gemm(const char * out_name, const std::vector & srcs) { return fuse(gemm_, out_name, srcs); } diff --git a/src/loader.h b/src/loader.h index f7264cf..0ae28e7 100644 --- a/src/loader.h +++ b/src/loader.h @@ -40,6 +40,10 @@ class WeightLoader { ggml_tensor * gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); ggml_tensor * f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); + // Explicit resident type, for weights that are neither a plain GEMM input + // nor F32 (BitVLA's int2-packed ternary blocks). + ggml_tensor * typed(ggml_type want, const char * fmt, ...) __attribute__((format(printf, 3, 4))); + // A miss is not an error. ggml_tensor * opt_gemm(const char * fmt, ...) __attribute__((format(printf, 2, 3))); ggml_tensor * opt_f32 (const char * fmt, ...) __attribute__((format(printf, 2, 3))); diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index 05347c3..b5a3e1e 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "loader.h" #include "options.h" #include "model.h" @@ -576,18 +577,10 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(bitvla): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(bitvla): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); - return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; - - auto mk_bit = [&](const char * name) { return mk(name, m->packed_int2 ? GGML_TYPE_I8 : m->matmul_type); }; + WeightLoader L("bitvla", g, m->ctx_weights, m->matmul_type); + auto mk_mm = [&](const char * name) { return L.gemm("%s", name); }; + auto mk_f32 = [&](const char * name) { return L.f32 ("%s", name); }; + auto mk_bit = [&](const char * name) { return L.typed(m->packed_int2 ? GGML_TYPE_I8 : m->matmul_type, "%s", name); }; bool ok = true; @@ -654,15 +647,8 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->ah_ln2_w&&m->ah_ln2_b&&m->ah_fc2_w&&m->ah_fc2_b; if (!ok) { std::fprintf(stderr, "vla(bitvla): weight tensor setup failed\n"); return nullptr; } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(bitvla): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(bitvla): failed to load %s (%zu vs %zu bytes)\n", ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + std::printf("vla(bitvla): weights resident in %.2f GiB (%s); image_id=%d proprio_id=%d action_begin_id=%d stop_id=%d\n", ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), m->packed_int2 ? "int2-packed + F32 sidecars" diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 26bf10a..269b4cd 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "loader.h" #include "options.h" #include "model.h" @@ -389,17 +390,11 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(evo1): ggml_init(ctx_weights) failed\n"); return nullptr; } - ggml_context * W = m->ctx_weights; - - auto mk = [&](const char * name, ggml_type type) -> ggml_tensor * { - const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(evo1): missing tensor %s\n", name); return nullptr; } - ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), ggml_n_dims(gt), gt->ne); - ggml_set_name(t, name); - return t; - }; - auto mk_mm = [&](const char * name) { return mk(name, m->matmul_type); }; - auto mk_f32 = [&](const char * name) { return mk(name, GGML_TYPE_F32); }; + // The vision tower is optional here, so misses are reported by the ok chain + // below rather than by the loader. + WeightLoader L("evo1", g, m->ctx_weights, m->matmul_type); + auto mk_mm = [&](const char * name) { return L.opt_gemm("%s", name); }; + auto mk_f32 = [&](const char * name) { return L.opt_f32 ("%s", name); }; bool ok = true; m->lm_output_norm = mk_f32("vlm.output_norm.weight"); ok &= (m->lm_output_norm != nullptr); @@ -445,7 +440,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->head_W2 && m->head_b2 && m->time_pos && m->state_W1 && m->state_b1 && m->state_W2 && m->state_b2; if (g.meta("vit.patch_embd.weight") && ok) { - m->vit_patch_w = mk("vit.patch_embd.weight", GGML_TYPE_F32); + m->vit_patch_w = mk_f32("vit.patch_embd.weight"); m->vit_patch_b = mk_f32("vit.patch_embd.bias"); m->vit_cls = mk_f32("vit.class_embd"); m->vit_pos = mk_f32("vit.pos_embd"); @@ -472,19 +467,11 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, } if (!ok) { std::fprintf(stderr, "vla(evo1): weight tensor setup failed\n"); return nullptr; } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); - if (!m->weight_buf) { std::fprintf(stderr, "vla(evo1): ggml_backend_alloc_ctx_tensors failed (OOM?)\n"); return nullptr; } - for (ggml_tensor * t = ggml_get_first_tensor(W); t; t = ggml_get_next_tensor(W, t)) { - std::vector bytes = g.read_convert(ggml_get_name(t), t->type); - if (bytes.empty() || bytes.size() != ggml_nbytes(t)) { - std::fprintf(stderr, "vla(evo1): failed to load %s (%zu vs %zu bytes)\n", - ggml_get_name(t), bytes.size(), ggml_nbytes(t)); return nullptr; - } - ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size()); - } + if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + std::printf("vla(evo1): weights resident in %.2f GiB (%s)%s\n", ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), - m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16", + dtype_name(m->matmul_type), m->have_vision ? " - incl. InternViT vision tower" : " - vision tower NOT loaded (precomputed_img_emb required)"); m->state_min = g.read_f32("state_min"); From 0b461540ad8af375fc7a6ef6b90da7069e8e3f87 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 22:39:23 +0700 Subject: [PATCH 13/21] share the siglip block weight struct with smolvla --- src/models/smolvla.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index e86e073..67aa957 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -16,6 +16,7 @@ // distilled action-expert weights and force num_steps = 1 at the denoise loops. #include "arch.h" +#include "modules/encoder.h" #include "options.h" #include "model.h" #include "modules/preprocess.h" @@ -282,7 +283,6 @@ struct ExpertLayerW { }; // SigLIP-B/16 vision block weights (SmolVLM2 tower, built in-tree). -struct SigLipLayerW { ggml_tensor *ln1w,*ln1b,*ln2w,*ln2b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wfc1,*bfc1,*Wfc2,*bfc2; }; } @@ -298,7 +298,7 @@ struct SmolVLAModelArch : public ModelArchBase { float vit_ln_eps = 1e-6f; ggml_tensor * vit_patch_w = nullptr, * vit_patch_b = nullptr, * vit_pos = nullptr; ggml_tensor * vit_post_ln_w = nullptr, * vit_post_ln_b = nullptr, * mm_fc = nullptr; - std::vector vit; + std::vector vit; ggml_backend_t backend = nullptr; ggml_backend_buffer_t weight_buf = nullptr; @@ -367,7 +367,7 @@ namespace { // One pre-norm SigLIP encoder block (SmolVLM2 tower), same graph as the other // in-tree models. Bidirectional attention, F32 score accumulation, tanh GELU. -ggml_tensor * build_siglip_layer(ggml_context * C, const SigLipLayerW & w, ggml_tensor * x, +ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { const float scale = 1.0f / std::sqrt((float) head_dim); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); @@ -1076,7 +1076,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, pending_f32.push_back({std::string(VP) + "post_layernorm.bias", m->vit_post_ln_b, {H}}); m->vit.resize(m->vit_layers); for (int64_t i = 0; i < m->vit_layers; ++i) { - SigLipLayerW & w = m->vit[i]; + EncBlockW & w = m->vit[i]; char pb[256]; std::snprintf(pb, sizeof(pb), "%sencoder.layers.%lld.", VP, (long long) i); const std::string pf = pb; w.ln1w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H); w.ln1b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H); From dfdb5851f11b309985e94694434d770e19914a32 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 22:43:55 +0700 Subject: [PATCH 14/21] drop the weight structs gr00t n1.7 no longer defines locally --- src/models/gr00tn1d7.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 4a0eb89..7c0d996 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -52,9 +52,6 @@ namespace { struct VlsaLayerW { ggml_tensor *n1w,*n1b,*n3w,*n3b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; }; -struct Qwen3LayerW { ggml_tensor *attn_norm,*Wq,*Wk,*Wv,*Wo,*q_norm,*k_norm,*ffn_norm,*Wgate,*Wup,*Wdown; }; -struct DitLayerW { ggml_tensor *adaln_w,*adaln_b,*Wq,*bq,*Wk,*bk,*Wv,*bv,*Wo,*bo,*Wff0,*bff0,*Wff2,*bff2; - ggml_tensor *Wqkv=nullptr,*bqkv=nullptr,*Wkv=nullptr,*bkv=nullptr; }; } From 8f70e609a29ad61081323791524570894255029a Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 23:17:06 +0700 Subject: [PATCH 15/21] one statement per line across src --- scripts/restyle.py | 147 +++++++++++ src/arch.h | 3 +- src/backend.h | 6 +- src/cuda/vla_cuda_bf16.cu | 163 +++++++++---- src/env_flag.h | 3 +- src/gguf_reader.h | 97 ++++++-- src/kernels/bitvla/bitnet_kernels.cu | 6 +- src/kernels/bitvla/bitnet_kernels.h | 33 ++- src/kernels/bitvla/bitvla_fp32head_cuda.cu | 63 +++-- src/kernels/bitvla/bitvla_lm_cuda.cu | 131 +++++++--- src/kernels/bitvla/bitvla_vit_cuda.cu | 34 ++- src/loader.cpp | 11 +- src/loader.h | 8 +- src/model.cpp | 88 +++++-- src/models/bitvla.cpp | 271 +++++++++++++++------ src/models/dit_common.h | 15 +- src/models/evo1.cpp | 106 +++++--- src/models/gr00tn1d5.cpp | 45 ++-- src/models/gr00tn1d6.cpp | 70 ++++-- src/models/gr00tn1d7.cpp | 219 +++++++++++++---- src/models/openvla_oft.cpp | 98 ++++++-- src/models/pi0.cpp | 87 +++++-- src/models/pi05.cpp | 97 ++++++-- src/models/smolvla.cpp | 214 +++++++++++----- src/models/vla_adapter.cpp | 113 ++++++--- src/models/vla_jepa.cpp | 204 ++++++++++++---- src/modules/dit_head.cpp | 12 +- src/modules/dual_tower.h | 15 +- src/modules/encoder.cpp | 6 +- src/modules/gemma_expert.h | 3 +- src/modules/preprocess.h | 9 +- src/modules/prompt.cpp | 21 +- src/modules/prompt.h | 8 +- src/modules/qwen3vl_vit.h | 31 ++- src/options.cpp | 137 ++++++++--- src/scratch_ctx.h | 60 +++-- src/serving/hf_fetch.h | 41 +++- src/serving/server.cpp | 43 ++-- src/serving/vla-bench.cpp | 64 +++-- src/serving/vla-cli.cpp | 123 +++++++--- src/serving/vlm-server.cpp | 30 ++- src/vla_c_api.cpp | 41 +++- src/vlm/engine.cpp | 15 +- 43 files changed, 2216 insertions(+), 775 deletions(-) create mode 100644 scripts/restyle.py diff --git a/scripts/restyle.py b/scripts/restyle.py new file mode 100644 index 0000000..06b8eb6 --- /dev/null +++ b/scripts/restyle.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics - Apache-2.0 +# +# Applies the structural half of the house style to C/C++ sources: one +# statement per line, no brace-wrapped one-liner blocks, no statement stapled +# to the end of an if/else. +# +# Whitespace only. Strings, character literals, comments and preprocessor lines +# are copied through untouched, and the splitter never runs inside parentheses, +# so a for-header's semicolons stay put. +# +# scripts/restyle.py src/models/vla_adapter.cpp ... + +import re +import sys + +def spans_to_skip(line): + """Character ranges of string/char literals and comments in `line`.""" + out, i, n = [], 0, len(line) + while i < n: + c = line[i] + if c in '"\'': + j = i + 1 + while j < n: + if line[j] == '\\': + j += 2 + continue + if line[j] == c: + break + j += 1 + out.append((i, min(j + 1, n))) + i = j + 1 + elif line.startswith('//', i): + out.append((i, n)) + break + elif line.startswith('/*', i): + j = line.find('*/', i + 2) + j = n if j < 0 else j + 2 + out.append((i, j)) + i = j + else: + i += 1 + return out + +def code_positions(line): + """Indices of `line` that are real code.""" + skip = spans_to_skip(line) + return [i for i in range(len(line)) if not any(a <= i < b for a, b in skip)] + +def split_statements(body, indent): + """`a; b; c` -> one statement per line, ignoring semicolons inside parens.""" + parts, depth, cur = [], 0, '' + live = set(code_positions(body)) + for i, ch in enumerate(body): + if i in live: + if ch in '([': + depth += 1 + elif ch in ')]': + depth -= 1 + elif ch == ';' and depth == 0: + parts.append(cur.strip() + ';') + cur = '' + continue + cur += ch + if cur.strip(): + parts.append(cur.strip()) + return [indent + p for p in parts if p.strip(';').strip()] + +ONE_LINE_BLOCK = re.compile(r'^(?P\s*)(?P.*?\{)\s*(?P[^{}]*?;)\s*\}\s*$') +CONTROL_HEAD = re.compile(r'^(?P\s*)(?:\}\s*else\s+)?(?:if|for|while)\s*\(') + +def control_body(line): + """Split `if/for/while (...) stmt;` into (head, body); None if it is not one.""" + m = CONTROL_HEAD.match(line) + if not m: + return None + live = set(code_positions(line)) + i = line.index('(', m.end() - 1) + depth = 0 + for j in range(i, len(line)): + if j not in live: + continue + if line[j] == '(': + depth += 1 + elif line[j] == ')': + depth -= 1 + if depth == 0: + head, body = line[:j + 1], line[j + 1:].strip() + if body and body.endswith(';') and '{' not in body and ';' not in body[:-1]: + return head, body + return None + return None + +def restyle_line(line): + if line.lstrip().startswith('#'): + return [line] + if '"' in line or "'" in line: + # A literal containing a brace or semicolon would confuse the splitters. + lit = ''.join(line[a:b] for a, b in spans_to_skip(line)) + if any(c in lit for c in '{};'): + return [line] + + m = ONE_LINE_BLOCK.match(line) + if m and 'namespace' not in m.group('head'): + inner = m.group('indent') + ' ' + stmts = split_statements(m.group('body'), inner) + if len(stmts) >= 1: + return [m.group('indent') + m.group('head')] + stmts + [m.group('indent') + '}'] + + m = re.match(r'^(?P\s*)(?:\}\s*)?else\s+(?P[^{};]*?;)\s*$', line) + if m and not re.match(r'^\s*(?:\}\s*)?else\s+if\b', line): + head = line[:line.index('else') + 4] + return [head.rstrip(), m.group('indent') + ' ' + m.group('body').strip()] + + cb = control_body(line) + if cb: + head, body = cb + indent = re.match(r'\s*', line).group(0) + return [head, indent + ' ' + body] + + return [line] + +def restyle(text): + total = 0 + for _ in range(6): + out, changed = [], 0 + for line in text.split('\n'): + new = restyle_line(line) + if new != [line]: + changed += 1 + out.extend(new) + text = '\n'.join(out) + total += changed + if not changed: + break + return text, total + +if __name__ == '__main__': + total = 0 + for path in sys.argv[1:]: + src = open(path).read() + dst, n = restyle(src) + if n: + open(path, 'w').write(dst) + print(f'{path}: {n} lines expanded') + total += n + print(f'total {total}') diff --git a/src/arch.h b/src/arch.h index 4a10b34..cdf1f63 100644 --- a/src/arch.h +++ b/src/arch.h @@ -44,7 +44,8 @@ inline int default_cpu_threads() { if (const char * e = std::getenv("VLA_N_THREADS")) { char * end = nullptr; const long n = std::strtol(e, &end, 10); - if (*end == '\0' && n > 0 && n <= 1024) return (int) n; + if (*end == '\0' && n > 0 && n <= 1024) + return (int) n; std::fprintf(stderr, "vla: ignoring VLA_N_THREADS='%s'\n", e); } const unsigned hw = std::thread::hardware_concurrency(); diff --git a/src/backend.h b/src/backend.h index 597275b..33377d4 100644 --- a/src/backend.h +++ b/src/backend.h @@ -57,7 +57,8 @@ namespace vla { inline void setenv_default(const char * key, const char * val) { #ifdef _WIN32 size_t len = 0; - if (getenv_s(&len, nullptr, 0, key) == 0 && len > 0) return; + if (getenv_s(&len, nullptr, 0, key) == 0 && len > 0) + return; _putenv_s(key, val); #else setenv(key, val, /*overwrite=*/0); @@ -79,7 +80,8 @@ struct Backend { /// silently read as device 0. inline int backend_device_index() { const char * e = std::getenv("VLA_DEVICE"); - if (!e || !*e) return 0; + if (!e || !*e) + return 0; char * end = nullptr; const long idx = std::strtol(e, &end, 10); if (*end != '\0' || idx < 0 || idx > 1024) { diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index 6afc426..d9dafe0 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -58,8 +58,12 @@ namespace { constexpr int BLOCK = 256; -inline __device__ float bf2f(const __nv_bfloat16 v) { return __bfloat162float(v); } -inline __device__ __nv_bfloat16 f2bf(const float v) { return __float2bfloat16(v); } +inline __device__ float bf2f(const __nv_bfloat16 v) { + return __bfloat162float(v); +} +inline __device__ __nv_bfloat16 f2bf(const float v) { + return __float2bfloat16(v); +} // --------------------------------------------------------------------------- // elementwise binary: dst = op(src0, src1), src1 broadcast per ggml_can_repeat @@ -80,8 +84,10 @@ inline __device__ float apply_bin(BinOp op, float a, float b) { // arguments so the branch is uniform across the block and the 64-bit modulo is // left for the general case that never fires in practice. inline __device__ int64_t bcast_idx(int64_t i, int64_t ne_src, int64_t ne_dst) { - if (ne_src == ne_dst) return i; - if (ne_src == 1) return 0; + if (ne_src == ne_dst) + return i; + if (ne_src == 1) + return 0; return i % ne_src; } @@ -191,7 +197,9 @@ __global__ void k_bin_bcast_bf16_flat( } // element strides (ggml stores byte strides) -inline int64_t es(const ggml_tensor * t, int i) { return t->nb[i] / ggml_type_size(t->type); } +inline int64_t es(const ggml_tensor * t, int i) { + return t->nb[i] / ggml_type_size(t->type); +} // Launch shape for the row-addressed kernels; ok=false means fall back to flat. struct RowGrid { @@ -211,13 +219,19 @@ inline bool force_flat() { inline RowGrid row_grid(int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { RowGrid g{}; const int64_t nz = ne2*ne3; - if (ne1 > 65535 || nz > 65535 || ne1 < 1 || nz < 1) { g.ok = false; return g; } + if (ne1 > 65535 || nz > 65535 || ne1 < 1 || nz < 1) { + g.ok = false; + return g; + } unsigned bx = 32; - while (bx < (unsigned) BLOCK && (int64_t) bx < ne0) bx *= 2; + while (bx < (unsigned) BLOCK && (int64_t) bx < ne0) + bx *= 2; int64_t gx = (ne0 + bx - 1) / bx; - if (gx > 65535) gx = 65535; - if (gx < 1) gx = 1; + if (gx > 65535) + gx = 65535; + if (gx < 1) + gx = 1; g.block = dim3(bx, 1, 1); g.grid = dim3((unsigned) gx, (unsigned) ne1, (unsigned) nz); @@ -229,11 +243,16 @@ template bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - if (!src0 || !src1) return false; - if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; - if (src1->type != GGML_TYPE_BF16 && src1->type != GGML_TYPE_F32) return false; - if (!ggml_are_same_shape(src0, dst)) return false; - if (!ggml_can_repeat(src1, src0)) return false; + if (!src0 || !src1) + return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) + return false; + if (src1->type != GGML_TYPE_BF16 && src1->type != GGML_TYPE_F32) + return false; + if (!ggml_are_same_shape(src0, dst)) + return false; + if (!ggml_can_repeat(src1, src0)) + return false; // Only the unfused path honours VLA_BF16_FLAT: the fused path has no flat // fallback, and declining there hands the run to ggml's aborting kernel. @@ -252,10 +271,13 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { if (vec8_shape) { const int64_t nvec = dst->ne[0] / 8; unsigned bx = 32; - while (bx < (unsigned) BLOCK && (int64_t) bx < nvec) bx *= 2; + while (bx < (unsigned) BLOCK && (int64_t) bx < nvec) + bx *= 2; int64_t gx = (nvec + bx - 1) / bx; - if (gx > 65535) gx = 65535; - if (gx < 1) gx = 1; + if (gx > 65535) + gx = 65535; + if (gx < 1) + gx = 1; const dim3 vgrid((unsigned) gx, g.grid.y, g.grid.z); const dim3 vblock(bx, 1, 1); @@ -269,8 +291,12 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { es(src1,1), es(src1,2), es(src1,3), \ es(dst,1), es(dst,2), es(dst,3)) - if (src1->type == GGML_TYPE_BF16) { VLA_LAUNCH_VEC8(__nv_bfloat16); } - else { VLA_LAUNCH_VEC8(float); } + if (src1->type == GGML_TYPE_BF16) { + VLA_LAUNCH_VEC8(__nv_bfloat16); + } + else { + VLA_LAUNCH_VEC8(float); + } #undef VLA_LAUNCH_VEC8 return true; } @@ -364,27 +390,37 @@ __global__ void k_fused_bin_bcast_bf16( template bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { - if (n_fuse < 2 || n_fuse > MAX_FUSE) return false; + if (n_fuse < 2 || n_fuse > MAX_FUSE) + return false; const ggml_tensor * src0 = dst->src[0]; - if (!src0) return false; - if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; - if (!ggml_are_same_shape(src0, dst)) return false; + if (!src0) + return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) + return false; + if (!ggml_are_same_shape(src0, dst)) + return false; // src[1] fixes the layout and type every other addend must match; the // upstream fusion check guarantees it, and this re-checks rather than // trusting it, because getting it wrong reads out of bounds. const ggml_tensor * src1 = dst->src[1]; - if (!src1) return false; - if (src1->type != GGML_TYPE_BF16 && src1->type != GGML_TYPE_F32) return false; - if (!ggml_can_repeat(src1, src0)) return false; + if (!src1) + return false; + if (src1->type != GGML_TYPE_BF16 && src1->type != GGML_TYPE_F32) + return false; + if (!ggml_can_repeat(src1, src0)) + return false; for (int k = 1; k < n_fuse; ++k) { const ggml_tensor * s = dst->src[k + 1]; - if (!s || s->type != src1->type) return false; - if (!ggml_are_same_shape(s, src1)) return false; + if (!s || s->type != src1->type) + return false; + if (!ggml_are_same_shape(s, src1)) + return false; for (int d = 0; d < GGML_MAX_DIMS; ++d) { - if (s->nb[d] != src1->nb[d]) return false; + if (s->nb[d] != src1->nb[d]) + return false; } } @@ -392,7 +428,8 @@ bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { // kernel, which aborts on BF16, so an unaddressable shape must decline // before the hook claims it. const RowGrid g = row_grid(dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); - if (!g.ok) return false; + if (!g.ok) + return false; #define VLA_LAUNCH_FUSED(TYPE) \ do { \ @@ -425,9 +462,12 @@ enum class UnOp { Silu, Relu, Gelu, GeluErf }; template inline __device__ float apply_unary(const float x) { - if (op == UnOp::Silu) return x / (1.0f + expf(-x)); - if (op == UnOp::Relu) return x > 0.0f ? x : 0.0f; - if (op == UnOp::GeluErf) return 0.5f*x*(1.0f + erff(x*0.70710678118654752440f)); + if (op == UnOp::Silu) + return x / (1.0f + expf(-x)); + if (op == UnOp::Relu) + return x > 0.0f ? x : 0.0f; + if (op == UnOp::GeluErf) + return 0.5f*x*(1.0f + erff(x*0.70710678118654752440f)); // tanh approximation, matching ggml's GGML_UNARY_OP_GELU const float c = 0.79788456080286535588f; // sqrt(2/pi) return 0.5f*x*(1.0f + tanhf(c*(x + 0.044715f*x*x*x))); @@ -445,9 +485,11 @@ __global__ void k_unary_bf16(const __nv_bfloat16 * __restrict__ x, template bool unary(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; - if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) + return false; // The elementwise index math above assumes a dense buffer. - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) return false; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) + return false; const int64_t n = ggml_nelements(dst); const int64_t blocks = (n + BLOCK - 1) / BLOCK; @@ -471,8 +513,10 @@ __global__ void k_scale_bf16(const __nv_bfloat16 * __restrict__ x, __nv_bfloat16 bool scale(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; - if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) + return false; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) + return false; float s = 1.0f, b = 0.0f; memcpy(&s, (const float *) dst->op_params + 0, sizeof(float)); @@ -495,7 +539,8 @@ __device__ inline float block_sum(float v, float * shared) { shared[tid] = v; __syncthreads(); for (int s = blockDim.x / 2; s > 0; s >>= 1) { - if (tid < s) shared[tid] += shared[tid + s]; + if (tid < s) + shared[tid] += shared[tid + s]; __syncthreads(); } return shared[0]; @@ -514,37 +559,45 @@ __global__ void k_norm_bf16(const __nv_bfloat16 * __restrict__ x, __nv_bfloat16 for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) { const float v = bf2f(xr[c]); sumsq += v*v; - if (!rms) sum += v; + if (!rms) + sum += v; } if (rms) { const float ms = block_sum(sumsq, shared) / (float) ncols; const float inv = rsqrtf(ms + eps); - for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) dr[c] = f2bf(bf2f(xr[c])*inv); + for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) + dr[c] = f2bf(bf2f(xr[c])*inv); } else { const float mean = block_sum(sum, shared) / (float) ncols; __syncthreads(); const float meansq = block_sum(sumsq, shared) / (float) ncols; const float inv = rsqrtf(meansq - mean*mean + eps); - for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) dr[c] = f2bf((bf2f(xr[c]) - mean)*inv); + for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) + dr[c] = f2bf((bf2f(xr[c]) - mean)*inv); } } template bool norm(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; - if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) return false; + if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16) + return false; // Rows must be dense; higher dims are handled by flattening into the row index. - if (src0->nb[0] != ggml_type_size(src0->type)) return false; - if (dst->nb[0] != ggml_type_size(dst->type)) return false; - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) return false; + if (src0->nb[0] != ggml_type_size(src0->type)) + return false; + if (dst->nb[0] != ggml_type_size(dst->type)) + return false; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) + return false; float eps = 0.0f; memcpy(&eps, dst->op_params, sizeof(float)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nelements(src0) / ncols; - if (nrows > 2147483647) return false; + if (nrows > 2147483647) + return false; k_norm_bf16<<<(int) nrows, BLOCK, 0, stream>>>( (const __nv_bfloat16 *) src0->data, (__nv_bfloat16 *) dst->data, @@ -571,7 +624,8 @@ cublasHandle_t g_handle = nullptr; bool mul_mat(ggml_tensor * dst, cudaStream_t stream) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - if (!src0 || !src1) return false; + if (!src0 || !src1) + return false; if (dst->type != GGML_TYPE_BF16 || src0->type != GGML_TYPE_BF16 || src1->type != GGML_TYPE_BF16) { return false; } @@ -581,10 +635,13 @@ bool mul_mat(ggml_tensor * dst, cudaStream_t stream) { // src0 is either shared across the whole batch or batched 1:1 with src1 const bool batch_ok = (src0->ne[2] == 1 && src0->ne[3] == 1) || (src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3]); - if (!batch_ok) return false; + if (!batch_ok) + return false; - if (!g_handle && cublasCreate(&g_handle) != CUBLAS_STATUS_SUCCESS) return false; - if (cublasSetStream(g_handle, stream) != CUBLAS_STATUS_SUCCESS) return false; + if (!g_handle && cublasCreate(&g_handle) != CUBLAS_STATUS_SUCCESS) + return false; + if (cublasSetStream(g_handle, stream) != CUBLAS_STATUS_SUCCESS) + return false; const int64_t ne00 = src0->ne[0], ne01 = src0->ne[1]; const int64_t ne10 = src1->ne[0], ne11 = src1->ne[1]; @@ -627,7 +684,8 @@ bool mul_mat(ggml_tensor * dst, cudaStream_t stream) { // --------------------------------------------------------------------------- extern "C" bool vla_cuda_bf16_fused_binbcast(ggml_tensor * dst, int n_fuse, void * stream_v) { - if (!dst) return false; + if (!dst) + return false; cudaStream_t stream = (cudaStream_t) stream_v; switch (dst->op) { @@ -638,7 +696,8 @@ extern "C" bool vla_cuda_bf16_fused_binbcast(ggml_tensor * dst, int n_fuse, void } extern "C" bool vla_cuda_bf16_forward(ggml_tensor * dst, void * stream_v) { - if (!dst) return false; + if (!dst) + return false; cudaStream_t stream = (cudaStream_t) stream_v; switch (dst->op) { diff --git a/src/env_flag.h b/src/env_flag.h index f7aa6ba..b2a0b41 100644 --- a/src/env_flag.h +++ b/src/env_flag.h @@ -42,7 +42,8 @@ namespace vla { */ inline bool env_flag(const char * name, bool def = false) { const char * v = std::getenv(name); - if (!v) return def; + if (!v) + return def; if (!*v) return false; // FOO= reads as "unset it" char buf[8] = {}; diff --git a/src/gguf_reader.h b/src/gguf_reader.h index ebe8944..23f8f26 100644 --- a/src/gguf_reader.h +++ b/src/gguf_reader.h @@ -39,9 +39,12 @@ struct gguf_reader { explicit gguf_reader(const char * arch_ = "vla") : arch(arch_) {} ~gguf_reader() { - if (fp) std::fclose(fp); - if (gctx) gguf_free(gctx); - if (meta_ctx) ggml_free(meta_ctx); + if (fp) + std::fclose(fp); + if (gctx) + gguf_free(gctx); + if (meta_ctx) + ggml_free(meta_ctx); } gguf_reader(const gguf_reader &) = delete; gguf_reader & operator=(const gguf_reader &) = delete; @@ -51,20 +54,29 @@ struct gguf_reader { p.no_alloc = true; p.ctx = &meta_ctx; gctx = gguf_init_from_file(path.c_str(), p); - if (!gctx) { std::fprintf(stderr, "vla(%s): gguf_init_from_file failed for %s\n", arch, path.c_str()); return false; } + if (!gctx) { + std::fprintf(stderr, "vla(%s): gguf_init_from_file failed for %s\n", arch, path.c_str()); + return false; + } fp = std::fopen(path.c_str(), "rb"); - if (!fp) { std::fprintf(stderr, "vla(%s): fopen failed for %s\n", arch, path.c_str()); return false; } + if (!fp) { + std::fprintf(stderr, "vla(%s): fopen failed for %s\n", arch, path.c_str()); + return false; + } data_off = gguf_get_data_offset(gctx); return true; } - bool has(const char * k) const { return gguf_find_key(gctx, k) >= 0; } + bool has(const char * k) const { + return gguf_find_key(gctx, k) >= 0; + } // gguf_get_val_* asserts on a type mismatch, killing the process on a bad // file. Check the declared type first. bool typed_key(const char * k, gguf_type want, int64_t * id_out) const { const int64_t id = gguf_find_key(gctx, k); - if (id < 0) return false; + if (id < 0) + return false; if (gguf_get_kv_type(gctx, id) != want) { std::fprintf(stderr, "vla(%s): key %s has unexpected type %d\n", arch, k, (int) gguf_get_kv_type(gctx, id)); @@ -74,11 +86,25 @@ struct gguf_reader { return true; } - uint32_t u32(const char * k) const { int64_t id; return typed_key(k, GGUF_TYPE_UINT32, &id) ? gguf_get_val_u32(gctx, id) : 0u; } - float f32(const char * k) const { int64_t id; return typed_key(k, GGUF_TYPE_FLOAT32, &id) ? gguf_get_val_f32(gctx, id) : 0.f; } - double f64(const char * k) const { int64_t id; return typed_key(k, GGUF_TYPE_FLOAT64, &id) ? gguf_get_val_f64(gctx, id) : 0.0; } - std::string str(const char * k) const { int64_t id; return typed_key(k, GGUF_TYPE_STRING, &id) ? std::string(gguf_get_val_str(gctx, id)) : std::string(); } - const ggml_tensor * meta(const char * name) const { return ggml_get_tensor(meta_ctx, name); } + uint32_t u32(const char * k) const { + int64_t id; + return typed_key(k, GGUF_TYPE_UINT32, &id) ? gguf_get_val_u32(gctx, id) : 0u; + } + float f32(const char * k) const { + int64_t id; + return typed_key(k, GGUF_TYPE_FLOAT32, &id) ? gguf_get_val_f32(gctx, id) : 0.f; + } + double f64(const char * k) const { + int64_t id; + return typed_key(k, GGUF_TYPE_FLOAT64, &id) ? gguf_get_val_f64(gctx, id) : 0.0; + } + std::string str(const char * k) const { + int64_t id; + return typed_key(k, GGUF_TYPE_STRING, &id) ? std::string(gguf_get_val_str(gctx, id)) : std::string(); + } + const ggml_tensor * meta(const char * name) const { + return ggml_get_tensor(meta_ctx, name); + } // Resident type for a weight: keep a quantized source type (Q8_0, Q4_0, ...) // so it stays packed and ggml_mul_mat dequantizes at compute; otherwise use @@ -92,7 +118,10 @@ struct gguf_reader { // partially written. bool read_raw(const char * name, void * buf, size_t cap) { const int64_t id = gguf_find_tensor(gctx, name); - if (id < 0) { std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch, name); return false; } + if (id < 0) { + std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch, name); + return false; + } const size_t off = data_off + gguf_get_tensor_offset(gctx, id); const size_t nb = gguf_get_tensor_size(gctx, id); if (nb != cap) { @@ -100,7 +129,8 @@ struct gguf_reader { arch, name, nb, cap); return false; } - if (fseeko(fp, (off_t) off, SEEK_SET) != 0) return false; + if (fseeko(fp, (off_t) off, SEEK_SET) != 0) + return false; return std::fread(buf, 1, nb, fp) == nb; } @@ -137,15 +167,29 @@ struct gguf_reader { if (f.empty()) return {}; const int64_t n = (int64_t) f.size(); if (gemma_norm) for (int64_t i = 0; i < n; ++i) f[i] += 1.0f; - if (target == GGML_TYPE_F32) { std::vector o(n * sizeof(float)); std::memcpy(o.data(), f.data(), o.size()); return o; } - if (target == GGML_TYPE_BF16) { std::vector o(n * sizeof(ggml_bf16_t)); ggml_fp32_to_bf16_row(f.data(), reinterpret_cast(o.data()), n); return o; } + if (target == GGML_TYPE_F32) { + std::vector o(n * sizeof(float)); + std::memcpy(o.data(), f.data(), o.size()); + return o; + } + if (target == GGML_TYPE_BF16) { + std::vector o(n * sizeof(ggml_bf16_t)); + ggml_fp32_to_bf16_row(f.data(), reinterpret_cast(o.data()), n); + return o; + } std::fprintf(stderr, "vla(%s): unsupported resident type %d for %s\n", arch, (int) target, name); return {}; } bool fetch_rows_f32(const char * name, const std::vector & row_ids, float * dst, int64_t cols) { const ggml_tensor * t = meta(name); - if (!t || t->ne[0] != cols || t->ne[2] != 1 || t->ne[3] != 1) { std::fprintf(stderr, "vla(%s): %s shape unfit for row-fetch\n", arch, name); return false; } - if (t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_BF16) { std::fprintf(stderr, "vla(%s): %s type %d not f32/bf16 for row-fetch\n", arch, name, (int) t->type); return false; } + if (!t || t->ne[0] != cols || t->ne[2] != 1 || t->ne[3] != 1) { + std::fprintf(stderr, "vla(%s): %s shape unfit for row-fetch\n", arch, name); + return false; + } + if (t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_BF16) { + std::fprintf(stderr, "vla(%s): %s type %d not f32/bf16 for row-fetch\n", arch, name, (int) t->type); + return false; + } const int64_t rows = t->ne[1]; const int64_t id = gguf_find_tensor(gctx, name); const size_t base = data_off + gguf_get_tensor_offset(gctx, id); @@ -154,11 +198,18 @@ struct gguf_reader { std::vector row(rb); for (size_t k = 0; k < row_ids.size(); ++k) { const int32_t r = row_ids[k]; - if (r < 0 || r >= rows) { std::fprintf(stderr, "vla(%s): row %d out of range for %s\n", arch, r, name); return false; } - if (fseeko(fp, (off_t) (base + (size_t) r * rb), SEEK_SET) != 0) return false; - if (std::fread(row.data(), 1, rb, fp) != rb) return false; - if (elsz == 4) std::memcpy(dst + k * cols, row.data(), rb); - else ggml_bf16_to_fp32_row(reinterpret_cast(row.data()), dst + k * cols, cols); + if (r < 0 || r >= rows) { + std::fprintf(stderr, "vla(%s): row %d out of range for %s\n", arch, r, name); + return false; + } + if (fseeko(fp, (off_t) (base + (size_t) r * rb), SEEK_SET) != 0) + return false; + if (std::fread(row.data(), 1, rb, fp) != rb) + return false; + if (elsz == 4) + std::memcpy(dst + k * cols, row.data(), rb); + else + ggml_bf16_to_fp32_row(reinterpret_cast(row.data()), dst + k * cols, cols); } return true; } diff --git a/src/kernels/bitvla/bitnet_kernels.cu b/src/kernels/bitvla/bitnet_kernels.cu index fe282b6..2204474 100644 --- a/src/kernels/bitvla/bitnet_kernels.cu +++ b/src/kernels/bitvla/bitnet_kernels.cu @@ -83,7 +83,8 @@ extern "C" void bitlinear_int8xint2_m( #define WIDE(NN, KK, WS) \ launch_ladder_int8xint2_m_wide( \ input0, input1, output0, s, ws, M, stream) - if (N == 2560 && K == 2560) WIDE(2560, 2560, 1); + if (N == 2560 && K == 2560) + WIDE(2560, 2560, 1); else if (N == 640 && K == 2560) WIDE(640, 2560, 1); else if (N == 13824 && K == 2560) WIDE(13824, 2560, 2); else if (N == 2560 && K == 6912) WIDE(2560, 6912, 1); @@ -99,7 +100,8 @@ extern "C" void bitlinear_int8xint2_m( return; } - if (N == 2560 && K == 2560) launch_ladder_int8xint2_m<2560, 2560, 1, 128>(input0, input1, output0, s, ws, M, stream); + if (N == 2560 && K == 2560) + launch_ladder_int8xint2_m<2560, 2560, 1, 128>(input0, input1, output0, s, ws, M, stream); else if (N == 640 && K == 2560) launch_ladder_int8xint2_m<640, 2560, 1, 128>(input0, input1, output0, s, ws, M, stream); else if (N == 13824 && K == 2560) launch_ladder_int8xint2_m<13824, 2560, 2, 128>(input0, input1, output0, s, ws, M, stream); else if (N == 2560 && K == 6912) launch_ladder_int8xint2_m<2560, 6912, 1, 128>(input0, input1, output0, s, ws, M, stream); diff --git a/src/kernels/bitvla/bitnet_kernels.h b/src/kernels/bitvla/bitnet_kernels.h index 73e114c..6420a81 100644 --- a/src/kernels/bitvla/bitnet_kernels.h +++ b/src/kernels/bitvla/bitnet_kernels.h @@ -191,7 +191,8 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( wmma::fragment b_frag; wmma::fragment acc[TILES_PER_WARP]; #pragma unroll - for (int t = 0; t < TILES_PER_WARP; ++t) wmma::fill_fragment(acc[t], 0); + for (int t = 0; t < TILES_PER_WARP; ++t) + wmma::fill_fragment(acc[t], 0); for (int k_0 = 0; k_0 < K / K_CHUNK; ++k_0) { #pragma unroll @@ -333,7 +334,8 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( wmma::fragment b_frag; wmma::fragment acc[M_PER_WARP]; #pragma unroll - for (int t = 0; t < M_PER_WARP; ++t) wmma::fill_fragment(acc[t], 0); + for (int t = 0; t < M_PER_WARP; ++t) + wmma::fill_fragment(acc[t], 0); for (int k_0 = 0; k_0 < K / K_CHUNK; ++k_0) { #pragma unroll @@ -379,7 +381,8 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( __syncthreads(); } - if (!my_tile_valid) return; + if (!my_tile_valid) + return; const int n_base = my_tile * 16; const float wsv = ws[n_base / (N / ws_num)]; @@ -474,37 +477,45 @@ __global__ void act_quant_kernel( float local_max = 0.0f; for (int k = tid; k < K; k += BLOCK_THREADS) { float v = fabsf(__bfloat162float(row_in[k])); - if (v > local_max) local_max = v; + if (v > local_max) + local_max = v; } for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, local_max, off); - if (other > local_max) local_max = other; + if (other > local_max) + local_max = other; } __shared__ float smem[32]; const int warp_id = tid >> 5; const int lane = tid & 31; - if (lane == 0) smem[warp_id] = local_max; + if (lane == 0) + smem[warp_id] = local_max; __syncthreads(); if (warp_id == 0) { float v = (tid < (BLOCK_THREADS + 31) / 32) ? smem[lane] : 0.0f; for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); - if (other > v) v = other; + if (other > v) + v = other; } - if (lane == 0) smem[0] = v; + if (lane == 0) + smem[0] = v; } __syncthreads(); const float amax = smem[0] < 1e-5f ? 1e-5f : smem[0]; const float scale = 127.0f / amax; - if (tid == 0) scales[m] = scale; + if (tid == 0) + scales[m] = scale; for (int k = tid; k < K; k += BLOCK_THREADS) { float v = __bfloat162float(row_in[k]) * scale; float q = nearbyintf(v); - if (q > 127.0f) q = 127.0f; - if (q < -128.0f) q = -128.0f; + if (q > 127.0f) + q = 127.0f; + if (q < -128.0f) + q = -128.0f; row_out[k] = (int8_t)q; } } diff --git a/src/kernels/bitvla/bitvla_fp32head_cuda.cu b/src/kernels/bitvla/bitvla_fp32head_cuda.cu index 38108c4..0dbb6c9 100644 --- a/src/kernels/bitvla/bitvla_fp32head_cuda.cu +++ b/src/kernels/bitvla/bitvla_fp32head_cuda.cu @@ -58,24 +58,34 @@ struct bitvla_fp32head_cuda_ctx { static float* upload_f32(const float* h, size_t n) { float* d = nullptr; cudaError_t e = cudaMalloc(&d, n * sizeof(float)); - if (e != cudaSuccess) { std::fprintf(stderr, "vla(bitvla_fp32head): cudaMalloc failed (%zu)\n", n); return nullptr; } + if (e != cudaSuccess) { + std::fprintf(stderr, "vla(bitvla_fp32head): cudaMalloc failed (%zu)\n", n); + return nullptr; + } e = cudaMemcpy(d, h, n * sizeof(float), cudaMemcpyHostToDevice); - if (e != cudaSuccess) { std::fprintf(stderr, "vla(bitvla_fp32head): cudaMemcpy H2D failed (%zu)\n", n); cudaFree(d); return nullptr; } + if (e != cudaSuccess) { + std::fprintf(stderr, "vla(bitvla_fp32head): cudaMemcpy H2D failed (%zu)\n", n); + cudaFree(d); + return nullptr; + } return d; } __global__ void gelu_erf_fp32_kernel(const float* __restrict__ in, float* __restrict__ out, int N) { const int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= N) return; + if (i >= N) + return; float x = in[i]; out[i] = 0.5f * x * (1.0f + erff(x * 0.70710678118654752440f)); } __global__ void relu_fp32_kernel(float* __restrict__ inout, int N) { const int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= N) return; + if (i >= N) + return; float v = inout[i]; - if (v < 0.0f) inout[i] = 0.0f; + if (v < 0.0f) + inout[i] = 0.0f; } template @@ -90,15 +100,20 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, float* o = out + (size_t)m * K; float sum = 0.0f; - for (int k = tid; k < K; k += BLOCK) sum += row[k]; - for (int off = 16; off > 0; off >>= 1) sum += __shfl_down_sync(0xffffffff, sum, off); + for (int k = tid; k < K; k += BLOCK) + sum += row[k]; + for (int off = 16; off > 0; off >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, off); __shared__ float smem[32]; - if ((tid & 31) == 0) smem[tid >> 5] = sum; + if ((tid & 31) == 0) + smem[tid >> 5] = sum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float mean = smem[0] / (float)K; @@ -108,13 +123,17 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, float v = row[k] - mean; vsum += v * v; } - for (int off = 16; off > 0; off >>= 1) vsum += __shfl_down_sync(0xffffffff, vsum, off); - if ((tid & 31) == 0) smem[tid >> 5] = vsum; + for (int off = 16; off > 0; off >>= 1) + vsum += __shfl_down_sync(0xffffffff, vsum, off); + if ((tid & 31) == 0) + smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float inv_std = rsqrtf(smem[0] / (float)K + eps); @@ -128,14 +147,16 @@ __global__ void add_bias_fp32_kernel(const float* x, const float* bias, float* out, int M, int K) { const int m = blockIdx.x; const int k = blockIdx.y * blockDim.x + threadIdx.x; - if (k >= K) return; + if (k >= K) + return; const size_t i = (size_t)m * K + k; out[i] = x[i] + bias[k]; } __global__ void add_fp32_kernel(const float* a, const float* b, float* out, int N) { const int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= N) return; + if (i >= N) + return; out[i] = a[i] + b[i]; } @@ -358,7 +379,8 @@ extern "C" int bitvla_fp32head_action_forward( } extern "C" void bitvla_fp32head_cuda_free(bitvla_fp32head_cuda_ctx* ctx) { - if (!ctx) return; + if (!ctx) + return; float* ws[] = { ctx->pp_fc1_w, ctx->pp_fc1_b, ctx->pp_fc2_w, ctx->pp_fc2_b, ctx->ah_ln1_w, ctx->ah_ln1_b, ctx->ah_fc1_w, ctx->ah_fc1_b, @@ -368,7 +390,10 @@ extern "C" void bitvla_fp32head_cuda_free(bitvla_fp32head_cuda_ctx* ctx) { ctx->d_state, ctx->d_pp_h1, ctx->d_pp_out, ctx->d_ah_in, ctx->d_ah_norm_big, ctx->d_ah_h, ctx->d_ah_tmp, ctx->d_ah_tmp2, ctx->d_ah_out, }; - for (float* p : ws) if (p) cudaFree(p); - if (ctx->cublas) cublasDestroy(ctx->cublas); + for (float* p : ws) + if (p) + cudaFree(p); + if (ctx->cublas) + cublasDestroy(ctx->cublas); delete ctx; } diff --git a/src/kernels/bitvla/bitvla_lm_cuda.cu b/src/kernels/bitvla/bitvla_lm_cuda.cu index 0e4c455..2ff5401 100644 --- a/src/kernels/bitvla/bitvla_lm_cuda.cu +++ b/src/kernels/bitvla/bitvla_lm_cuda.cu @@ -50,14 +50,18 @@ __global__ void rmsnorm_bf16_kernel(const __nv_bfloat16* __restrict__ x, float v = __bfloat162float(row[k]); ss += v * v; } - for (int off = 16; off > 0; off >>= 1) ss += __shfl_down_sync(0xffffffff, ss, off); + for (int off = 16; off > 0; off >>= 1) + ss += __shfl_down_sync(0xffffffff, ss, off); __shared__ float smem[32]; - if ((tid & 31) == 0) smem[tid >> 5] = ss; + if ((tid & 31) == 0) + smem[tid >> 5] = ss; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float mean = smem[0] / (float)K; @@ -104,22 +108,27 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, float mx = -INFINITY; for (int i = tid; i < S; i += BLOCK) { float v = __bfloat162float(r[i]) * scale; - if (v > mx) mx = v; + if (v > mx) + mx = v; } for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, mx, off); - if (other > mx) mx = other; + if (other > mx) + mx = other; } __shared__ float smem[32]; - if ((tid & 31) == 0) smem[tid >> 5] = mx; + if ((tid & 31) == 0) + smem[tid >> 5] = mx; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : -INFINITY; for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); - if (other > v) v = other; + if (other > v) + v = other; } - if (tid == 0) smem[0] = v; + if (tid == 0) + smem[0] = v; } __syncthreads(); const float max_v = smem[0]; @@ -128,13 +137,17 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, for (int i = tid; i < S; i += BLOCK) { s_sum += expf(__bfloat162float(r[i]) * scale - max_v); } - for (int off = 16; off > 0; off >>= 1) s_sum += __shfl_down_sync(0xffffffff, s_sum, off); - if ((tid & 31) == 0) smem[tid >> 5] = s_sum; + for (int off = 16; off > 0; off >>= 1) + s_sum += __shfl_down_sync(0xffffffff, s_sum, off); + if ((tid & 31) == 0) + smem[tid >> 5] = s_sum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float inv_sum = 1.0f / smem[0]; @@ -149,16 +162,19 @@ __global__ void squared_relu_mul_bf16_kernel(const __nv_bfloat16* g, const __nv_bfloat16* u, __nv_bfloat16* out, int N) { const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); - if (i >= N) return; + if (i >= N) + return; float gv = __bfloat162float(g[i]); - if (gv < 0.0f) gv = 0.0f; + if (gv < 0.0f) + gv = 0.0f; out[i] = __float2bfloat16(gv * gv * __bfloat162float(u[i])); } __global__ void add_bf16_kernel(const __nv_bfloat16* a, const __nv_bfloat16* b, __nv_bfloat16* out, int N) { const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); - if (i >= N) return; + if (i >= N) + return; out[i] = __float2bfloat16(__bfloat162float(a[i]) + __bfloat162float(b[i])); } @@ -308,7 +324,11 @@ extern "C" bitvla_lm_cuda_ctx* bitvla_lm_cuda_init(int hidden, int n_q, int n_kv ctx->layers.resize(n_layers); cublasStatus_t cbs = cublasCreate(&ctx->cublas); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_lm_cuda): cublasCreate failed\n"); delete ctx; return nullptr; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_lm_cuda): cublasCreate failed\n"); + delete ctx; + return nullptr; + } const int half = head_dim / 2; std::vector h_cos((size_t)max_seq * half), h_sin((size_t)max_seq * half); @@ -348,7 +368,8 @@ extern "C" bitvla_lm_cuda_ctx* bitvla_lm_cuda_init(int hidden, int n_q, int n_kv } extern "C" void bitvla_lm_cuda_free(bitvla_lm_cuda_ctx* ctx) { - if (!ctx) return; + if (!ctx) + return; cublasDestroy(ctx->cublas); cudaFree(ctx->d_cos); cudaFree(ctx->d_sin); cudaFree(ctx->d_h); cudaFree(ctx->d_h_norm); @@ -378,7 +399,8 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea const char* l0_dir = (L == 0) ? std::getenv("VLA_BITVLA_DUMP_L0") : nullptr; auto l0_dump = [&](const char* name, const __nv_bfloat16* d_ptr, size_t n) { - if (!l0_dir) return; + if (!l0_dir) + return; cudaStreamSynchronize(stream); std::vector<__nv_bfloat16> tmp(n); std::vector f32(n); @@ -391,7 +413,10 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea } std::string path = std::string(l0_dir) + "/" + name + ".bin"; FILE* f = std::fopen(path.c_str(), "wb"); - if (f) { std::fwrite(f32.data(), sizeof(float), n, f); std::fclose(f); } + if (f) { + std::fwrite(f32.data(), sizeof(float), n, f); + std::fclose(f); + } }; bitvla_rmsnorm_bf16(ctx->d_h, lr.attn_norm_w, ctx->d_h_norm, ctx->rms_eps, seq, hidden, stream); @@ -435,7 +460,10 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea ctx->d_scores,CUDA_R_16BF, seq, (long long)seq * seq, n_q, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_lm_cuda): QK^T gemm failed @L%d (%d)\n", L, cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_lm_cuda): QK^T gemm failed @L%d (%d)\n", L, cbs); + return -1; + } const float scl = 1.0f / std::sqrt((float)hd); bitvla_softmax_scaled_bf16(ctx->d_scores, scl, n_q * seq, seq, stream); @@ -450,7 +478,10 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea ctx->d_attn_out, CUDA_R_16BF, hd, (long long)seq * hd, n_q, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_lm_cuda): attn@V gemm failed @L%d (%d)\n", L, cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_lm_cuda): attn@V gemm failed @L%d (%d)\n", L, cbs); + return -1; + } bitvla_transpose_NshHd_to_sNhd_bf16(ctx->d_attn_out, ctx->d_attn_merged, n_q, seq, hd, stream); @@ -498,13 +529,15 @@ __global__ void gate_up_fused_sqrelu_mul_bf16_kernel(const __nv_bfloat16* __rest int seq, int ffn) { const int idx = (int)(blockIdx.x * blockDim.x + threadIdx.x); const int total = seq * ffn; - if (idx >= total) return; + if (idx >= total) + return; const int s = idx / ffn; const int k = idx % ffn; const size_t row_base = (size_t)s * 2 * ffn; float g = __bfloat162float(gu[row_base + k]); float u = __bfloat162float(gu[row_base + ffn + k]); - if (g < 0.0f) g = 0.0f; + if (g < 0.0f) + g = 0.0f; out[(size_t)idx] = __float2bfloat16(g * g * u); } extern "C" void gate_up_fused_sqrelu_mul_bf16(const __nv_bfloat16* gu, __nv_bfloat16* out, @@ -528,15 +561,20 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, __nv_bfloat16* o = out + (size_t)m * K; float sum = 0.0f; - for (int k = tid; k < K; k += BLOCK) sum += __bfloat162float(row[k]); - for (int off = 16; off > 0; off >>= 1) sum += __shfl_down_sync(0xffffffff, sum, off); + for (int k = tid; k < K; k += BLOCK) + sum += __bfloat162float(row[k]); + for (int off = 16; off > 0; off >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, off); __shared__ float smem[32]; - if ((tid & 31) == 0) smem[tid >> 5] = sum; + if ((tid & 31) == 0) + smem[tid >> 5] = sum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float mean = smem[0] / (float)K; @@ -546,13 +584,17 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, float v = __bfloat162float(row[k]) - mean; vsum += v * v; } - for (int off = 16; off > 0; off >>= 1) vsum += __shfl_down_sync(0xffffffff, vsum, off); - if ((tid & 31) == 0) smem[tid >> 5] = vsum; + for (int off = 16; off > 0; off >>= 1) + vsum += __shfl_down_sync(0xffffffff, vsum, off); + if ((tid & 31) == 0) + smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); - if (tid == 0) smem[0] = v; + for (int off = 16; off > 0; off >>= 1) + v += __shfl_down_sync(0xffffffff, v, off); + if (tid == 0) + smem[0] = v; } __syncthreads(); const float inv_std = rsqrtf(smem[0] / (float)K + eps); @@ -567,7 +609,8 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, __global__ void gelu_tanh_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* out, int N) { const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); - if (i >= N) return; + if (i >= N) + return; float x = __bfloat162float(in[i]); const float kAlpha = 0.7978845608028654f; @@ -581,7 +624,8 @@ __global__ void add_bias_bf16_kernel(const __nv_bfloat16* x, const __nv_bfloat16 __nv_bfloat16* out, int M, int K) { const int m = (int)blockIdx.x; const int k = (int)(blockIdx.y * blockDim.x + threadIdx.x); - if (k >= K) return; + if (k >= K) + return; const size_t i = (size_t)m * K + k; out[i] = __float2bfloat16(__bfloat162float(x[i]) + __bfloat162float(bias[k])); } @@ -589,7 +633,8 @@ __global__ void add_bias_bf16_kernel(const __nv_bfloat16* x, const __nv_bfloat16 __global__ void zero_tail_bf16_kernel(__nv_bfloat16* x, int total_cols, int start_col) { const int m = (int)blockIdx.x; const int k = (int)(start_col + blockIdx.y * blockDim.x + threadIdx.x); - if (k >= total_cols) return; + if (k >= total_cols) + return; x[(size_t)m * total_cols + k] = __float2bfloat16(0.0f); } @@ -612,7 +657,8 @@ extern "C" void bitvla_add_bias_bf16(const __nv_bfloat16* x, const __nv_bfloat16 } extern "C" void bitvla_zero_tail_bf16(__nv_bfloat16* x, int M, int total_cols, int start_col, cudaStream_t stream) { - if (start_col >= total_cols) return; + if (start_col >= total_cols) + return; constexpr int B = 128; const int len = total_cols - start_col; const int n_kb = (len + B - 1) / B; @@ -633,7 +679,8 @@ extern "C" int bitvla_lm_cuda_forward(bitvla_lm_cuda_ctx* ctx, std::vector<__nv_bfloat16> h_dump; std::vector h_dump_f32; auto dump_to_file = [&](const char* name, const __nv_bfloat16* d_ptr) { - if (!dump_dir) return; + if (!dump_dir) + return; const size_t n = (size_t) seq * ctx->hidden; h_dump.resize(n); h_dump_f32.resize(n); cudaMemcpy(h_dump.data(), d_ptr, n * sizeof(__nv_bfloat16), cudaMemcpyDeviceToHost); @@ -645,7 +692,10 @@ extern "C" int bitvla_lm_cuda_forward(bitvla_lm_cuda_ctx* ctx, } std::string path = std::string(dump_dir) + "/" + name + ".bin"; FILE* f = std::fopen(path.c_str(), "wb"); - if (f) { std::fwrite(h_dump_f32.data(), sizeof(float), n, f); std::fclose(f); } + if (f) { + std::fwrite(h_dump_f32.data(), sizeof(float), n, f); + std::fclose(f); + } }; CUDA_OK(cudaMemcpyAsync(ctx->d_h, d_in, (size_t)seq * ctx->hidden * sizeof(__nv_bfloat16), @@ -656,7 +706,8 @@ extern "C" int bitvla_lm_cuda_forward(bitvla_lm_cuda_ctx* ctx, } for (int L = 0; L < ctx->n_layers; ++L) { int rc = run_layer(ctx, L, seq, stream); - if (rc != 0) return rc; + if (rc != 0) + return rc; if (dump_dir) { cudaStreamSynchronize(stream); char name[64]; std::snprintf(name, sizeof(name), "lm_layer_%d", L); diff --git a/src/kernels/bitvla/bitvla_vit_cuda.cu b/src/kernels/bitvla/bitvla_vit_cuda.cu index 08ae84d..2185d28 100644 --- a/src/kernels/bitvla/bitvla_vit_cuda.cu +++ b/src/kernels/bitvla/bitvla_vit_cuda.cu @@ -32,7 +32,8 @@ extern "C" void bitvla_act_quant_cuda(const __nv_bfloat16* in, int8_t* out, __global__ void gelu_erf_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* out, int N) { const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); - if (i >= N) return; + if (i >= N) + return; const float x = __bfloat162float(in[i]); const float inv_sqrt2 = 0.7071067811865475f; out[i] = __float2bfloat16(0.5f * x * (1.0f + erff(x * inv_sqrt2))); @@ -136,7 +137,8 @@ bitvla_vit_cuda_ctx* bitvla_vit_cuda_init(int n_layers, int hidden, int n_heads, } void bitvla_vit_cuda_free(bitvla_vit_cuda_ctx* ctx) { - if (!ctx) return; + if (!ctx) + return; cublasDestroy(ctx->cublas); cudaFree(ctx->d_h); cudaFree(ctx->d_h_norm); cudaFree(ctx->d_act_int8_h); cudaFree(ctx->d_act_int8_ffn); cudaFree(ctx->d_act_s); @@ -200,7 +202,10 @@ static int run_vit_layer(bitvla_vit_cuda_ctx* ctx, int L, cudaStream_t stream) { &beta, ctx->d_scores, CUDA_R_16BF, seq, (long long) seq * seq, n_heads, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_vit_cuda): QK^T gemm @L%d failed (%d)\n", L, cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_vit_cuda): QK^T gemm @L%d failed (%d)\n", L, cbs); + return -1; + } const float scl = 1.0f / std::sqrt((float) hd); bitvla_softmax_scaled_bf16(ctx->d_scores, scl, n_heads * seq, seq, stream); @@ -213,7 +218,10 @@ static int run_vit_layer(bitvla_vit_cuda_ctx* ctx, int L, cudaStream_t stream) { &beta, ctx->d_attn_out, CUDA_R_16BF, hd, (long long) seq * hd, n_heads, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_vit_cuda): attn@V gemm @L%d failed (%d)\n", L, cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_vit_cuda): attn@V gemm @L%d failed (%d)\n", L, cbs); + return -1; + } bitvla_transpose_NshHd_to_sNhd_bf16(ctx->d_attn_out, ctx->d_attn_merged, n_heads, seq, hd, stream); @@ -265,7 +273,10 @@ int bitvla_vit_cuda_forward(bitvla_vit_cuda_ctx* ctx, &beta, ctx->d_h, CUDA_R_16BF, H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_vit_cuda): patch_embed gemm failed (%d)\n", cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_vit_cuda): patch_embed gemm failed (%d)\n", cbs); + return -1; + } bitvla_add_bias_bf16(ctx->d_h, ctx->patch_b, ctx->d_h, seq, H, stream); @@ -273,7 +284,8 @@ int bitvla_vit_cuda_forward(bitvla_vit_cuda_ctx* ctx, for (int L = 0; L < ctx->n_layers; ++L) { int rc = run_vit_layer(ctx, L, stream); - if (rc != 0) return rc; + if (rc != 0) + return rc; } cbs = cublasGemmEx( @@ -284,7 +296,10 @@ int bitvla_vit_cuda_forward(bitvla_vit_cuda_ctx* ctx, &beta, ctx->d_mm_h1, CUDA_R_16BF, M, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_vit_cuda): MM linear_1 gemm failed (%d)\n", cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_vit_cuda): MM linear_1 gemm failed (%d)\n", cbs); + return -1; + } bitvla_add_bias_bf16(ctx->d_mm_h1, ctx->mm_b1, ctx->d_mm_h1, seq, M, stream); gelu_erf_bf16(ctx->d_mm_h1, ctx->d_mm_h1, seq * M, stream); @@ -296,7 +311,10 @@ int bitvla_vit_cuda_forward(bitvla_vit_cuda_ctx* ctx, &beta, d_out, CUDA_R_16BF, M, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - if (cbs != CUBLAS_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla_vit_cuda): MM linear_2 gemm failed (%d)\n", cbs); return -1; } + if (cbs != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(bitvla_vit_cuda): MM linear_2 gemm failed (%d)\n", cbs); + return -1; + } bitvla_add_bias_bf16(d_out, ctx->mm_b2, d_out, seq, M, stream); return 0; } diff --git a/src/loader.cpp b/src/loader.cpp index f547119..1c1f6ce 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -53,7 +53,8 @@ ggml_tensor * WeightLoader::declare(ggml_type want, bool required, bool gemma_no } ggml_set_name(t, name); - if (gemma_norm) gemma_norms_.push_back(name); + if (gemma_norm) + gemma_norms_.push_back(name); return t; } @@ -91,7 +92,10 @@ ggml_tensor * WeightLoader::fuse_f32(const char * out_name, const std::vector & srcs) { - if (srcs.empty()) { ok_ = false; return nullptr; } + if (srcs.empty()) { + ok_ = false; + return nullptr; + } const ggml_tensor * first = g_.meta(srcs[0].c_str()); if (!first) { @@ -141,7 +145,8 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu const char * name = ggml_get_name(t); const bool fused = std::any_of(fused_.begin(), fused_.end(), [&](const Fused & f) { return f.dst == t; }); - if (fused) continue; + if (fused) + continue; const bool gn = std::find(gemma_norms_.begin(), gemma_norms_.end(), name) != gemma_norms_.end(); diff --git a/src/loader.h b/src/loader.h index 0ae28e7..649c9e9 100644 --- a/src/loader.h +++ b/src/loader.h @@ -56,8 +56,12 @@ class WeightLoader { ggml_tensor * fuse_gemm(const char * out_name, const std::vector & srcs); ggml_tensor * fuse_f32 (const char * out_name, const std::vector & srcs); - ggml_type gemm_type() const { return gemm_; } - bool ok() const { return ok_; } + ggml_type gemm_type() const { + return gemm_; + } + bool ok() const { + return ok_; + } bool upload(ggml_backend_t backend, ggml_backend_buffer_t * out_buf); diff --git a/src/model.cpp b/src/model.cpp index e17a2f6..cd3eab1 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -33,7 +33,8 @@ struct Model { namespace { bool ends_with_gguf(const std::string& p) { - if (p.size() < 5) return false; + if (p.size() < 5) + return false; return std::strcmp(p.c_str() + p.size() - 5, ".gguf") == 0; } @@ -42,16 +43,20 @@ bool detect_arch_gguf(const std::string& path, Arch* out) { p.no_alloc = true; p.ctx = nullptr; gguf_context * gctx = gguf_init_from_file(path.c_str(), p); - if (!gctx) return false; + if (!gctx) + return false; auto try_str = [&](const char * key, std::string& val) -> bool { const int64_t kid = gguf_find_key(gctx, key); - if (kid < 0) return false; + if (kid < 0) + return false; // gguf_get_val_str asserts (aborts) if the key is not a string, so a // malformed GGUF would kill the process here. Fail the probe instead. - if (gguf_get_kv_type(gctx, kid) != GGUF_TYPE_STRING) return false; + if (gguf_get_kv_type(gctx, kid) != GGUF_TYPE_STRING) + return false; const char * s = gguf_get_val_str(gctx, kid); - if (!s) return false; + if (!s) + return false; val = s; return true; }; @@ -70,17 +75,50 @@ bool detect_arch_gguf(const std::string& path, Arch* out) { try_str("openvla_oft.architecture", arch_str) || try_str("vla_jepa.architecture", arch_str) || try_str("vla_adapter.architecture", arch_str)) { - if (arch_str == "smolvla") { *out = Arch::SMOLVLA; ok = true; } - else if (arch_str == "pi0") { *out = Arch::PI0; ok = true; } - else if (arch_str == "pi05") { *out = Arch::PI05; ok = true; } - else if (arch_str == "evo1") { *out = Arch::EVO1; ok = true; } - else if (arch_str == "gr00t_n1_5") { *out = Arch::GR00T_N1_5; ok = true; } - else if (arch_str == "gr00t_n1_6") { *out = Arch::GR00T_N1_6; ok = true; } - else if (arch_str == "gr00t_n1_7") { *out = Arch::GR00T_N1_7; ok = true; } - else if (arch_str == "bitvla") { *out = Arch::BITVLA; ok = true; } - else if (arch_str == "vla_adapter"){ *out = Arch::VLA_ADAPTER;ok = true; } - else if (arch_str == "openvla_oft"){ *out = Arch::OPENVLA_OFT;ok = true; } - else if (arch_str == "vla_jepa") { *out = Arch::VLA_JEPA; ok = true; } + if (arch_str == "smolvla") { + *out = Arch::SMOLVLA; + ok = true; + } + else if (arch_str == "pi0") { + *out = Arch::PI0; + ok = true; + } + else if (arch_str == "pi05") { + *out = Arch::PI05; + ok = true; + } + else if (arch_str == "evo1") { + *out = Arch::EVO1; + ok = true; + } + else if (arch_str == "gr00t_n1_5") { + *out = Arch::GR00T_N1_5; + ok = true; + } + else if (arch_str == "gr00t_n1_6") { + *out = Arch::GR00T_N1_6; + ok = true; + } + else if (arch_str == "gr00t_n1_7") { + *out = Arch::GR00T_N1_7; + ok = true; + } + else if (arch_str == "bitvla") { + *out = Arch::BITVLA; + ok = true; + } + else if (arch_str == "vla_adapter"){ + *out = Arch::VLA_ADAPTER; + ok = true; + } + else if (arch_str == "openvla_oft"){ + *out = Arch::OPENVLA_OFT; + ok = true; + } + else if (arch_str == "vla_jepa") { + *out = Arch::VLA_JEPA; + ok = true; + } } gguf_free(gctx); @@ -113,13 +151,16 @@ namespace { bool detect_arch_safetensors(const std::string& path, Arch* out) { std::ifstream f(path, std::ios::binary); - if (!f) return false; + if (!f) + return false; uint64_t header_size = 0; f.read(reinterpret_cast(&header_size), sizeof(header_size)); - if (!f || header_size == 0 || header_size > (1u << 28)) return false; + if (!f || header_size == 0 || header_size > (1u << 28)) + return false; std::string header(header_size, '\0'); f.read(header.data(), header_size); - if (!f) return false; + if (!f) + return false; if (header.find("vlm_with_expert.vlm.") != std::string::npos) { *out = Arch::SMOLVLA; @@ -143,8 +184,10 @@ bool detect_arch_safetensors(const std::string& path, Arch* out) { } bool detect_arch_from_ckpt(const std::string& ckpt_path, Arch* out) { - if (!out) return false; - if (ends_with_gguf(ckpt_path)) return detect_arch_gguf(ckpt_path, out); + if (!out) + return false; + if (ends_with_gguf(ckpt_path)) + return detect_arch_gguf(ckpt_path, out); return detect_arch_safetensors(ckpt_path, out); } @@ -217,7 +260,8 @@ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, impl = vla_jepa_create(mmproj_path, ckpt_path, config_path, opts); break; } - if (!impl) return nullptr; + if (!impl) + return nullptr; if (!config_is_sane(impl->cfg)) { std::fprintf(stderr, "vla: refusing to load %s\n", ckpt_path.c_str()); return nullptr; diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index b5a3e1e..d705acc 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -78,14 +78,18 @@ void bitvla_act_quant_op(ggml_tensor * dst, const ggml_tensor * a, int ith, int const float * row_in = src + r * cols; float * row_out = out + r * cols; float amax = 0.0f; - for (int64_t c = 0; c < cols; ++c) amax = std::max(amax, std::fabs(row_in[c])); - if (amax < 1e-5f) amax = 1e-5f; + for (int64_t c = 0; c < cols; ++c) + amax = std::max(amax, std::fabs(row_in[c])); + if (amax < 1e-5f) + amax = 1e-5f; const float s = 127.0f / amax; const float inv_s = 1.0f / s; for (int64_t c = 0; c < cols; ++c) { float q = std::nearbyintf(row_in[c] * s); - if (q > 127.0f) q = 127.0f; - if (q < -128.0f) q = -128.0f; + if (q > 127.0f) + q = 127.0f; + if (q < -128.0f) + q = -128.0f; row_out[c] = q * inv_s; } } @@ -246,7 +250,8 @@ bool parse_stats_json(const std::string & js, const char * env_key, while (i < where.size() && depth > 0) { if (where[i] == '{') depth++; else if (where[i] == '}') depth--; - if (depth == 0) break; + if (depth == 0) + break; i++; } return where.substr(s, i - s + 1); @@ -260,11 +265,14 @@ bool parse_stats_json(const std::string & js, const char * env_key, out.clear(); const char * s = inner.c_str(); while (*s) { - while (*s == ' ' || *s == '\t' || *s == '\n' || *s == ',') s++; - if (!*s) break; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == ',') + s++; + if (!*s) + break; char * end = nullptr; float v = std::strtof(s, &end); - if (end == s) break; + if (end == s) + break; out.push_back(v); s = end; } @@ -279,11 +287,20 @@ bool parse_stats_json(const std::string & js, const char * env_key, out.clear(); size_t i = 0; while (i < inner.size()) { - while (i < inner.size() && (inner[i] == ' ' || inner[i] == ',' || inner[i] == '\n' || inner[i] == '\t')) i++; - if (i >= inner.size()) break; - if (inner.compare(i, 4, "true") == 0) { out.push_back(1); i += 4; } - else if (inner.compare(i, 5, "false") == 0) { out.push_back(0); i += 5; } - else i++; + while (i < inner.size() && (inner[i] == ' ' || inner[i] == ',' || inner[i] == '\n' || inner[i] == '\t')) + i++; + if (i >= inner.size()) + break; + if (inner.compare(i, 4, "true") == 0) { + out.push_back(1); + i += 4; + } + else if (inner.compare(i, 5, "false") == 0) { + out.push_back(0); + i += 5; + } + else + i++; } return true; }; @@ -294,20 +311,30 @@ bool parse_stats_json(const std::string & js, const char * env_key, } else { size_t p = js.find('"'); - if (p == std::string::npos) return false; + if (p == std::string::npos) + return false; size_t q = js.find('"', p + 1); - if (q == std::string::npos) return false; + if (q == std::string::npos) + return false; suite = js.substr(p + 1, q - p - 1); } resolved_key = suite; const std::string suite_obj = find_obj(js, suite); - if (suite_obj.empty()) { std::fprintf(stderr, "vla(bitvla): suite key '%s' not found in statistics_json\n", suite.c_str()); return false; } + if (suite_obj.empty()) { + std::fprintf(stderr, "vla(bitvla): suite key '%s' not found in statistics_json\n", suite.c_str()); + return false; + } const std::string action_obj = find_obj(suite_obj, "action"); - if (action_obj.empty()) { std::fprintf(stderr, "vla(bitvla): no action stats under '%s'\n", suite.c_str()); return false; } + if (action_obj.empty()) { + std::fprintf(stderr, "vla(bitvla): no action stats under '%s'\n", suite.c_str()); + return false; + } - if (!parse_array_floats(action_obj, "q01", q01)) return false; - if (!parse_array_floats(action_obj, "q99", q99)) return false; + if (!parse_array_floats(action_obj, "q01", q01)) + return false; + if (!parse_array_floats(action_obj, "q99", q99)) + return false; if (!parse_array_bools (action_obj, "mask", mask)) { mask.assign(q01.size(), 1); @@ -405,16 +432,20 @@ static void recover_ternary_and_scale(const float* W, int64_t n, // Per-tensor absmean scale (1/mean|W|), matching scripts/bitvla_int2_pack.py; // the int2-packed path bakes the same scale. double s = 0.0; - for (int64_t i = 0; i < n; ++i) s += std::fabs((double) W[i]); + for (int64_t i = 0; i < n; ++i) + s += std::fabs((double) W[i]); float mean = n > 0 ? (float) (s / (double) n) : 0.0f; - if (mean < 1e-5f) mean = 1e-5f; + if (mean < 1e-5f) + mean = 1e-5f; absmean = mean; const float inv = 1.0f / mean; ternary.resize(n); for (int64_t i = 0; i < n; ++i) { float q = std::nearbyintf(W[i] * inv); - if (q > 1.0f) q = 1.0f; - if (q < -1.0f) q = -1.0f; + if (q > 1.0f) + q = 1.0f; + if (q < -1.0f) + q = -1.0f; ternary[i] = (int8_t) q; } } @@ -458,7 +489,8 @@ static inline uint16_t f32_to_bf16_u16(float f) { static __nv_bfloat16* upload_bf16_from_f32(const float* h, size_t n, std::vector& out_ptrs) { std::vector tmp(n); - for (size_t i = 0; i < n; ++i) tmp[i] = f32_to_bf16_u16(h[i]); + for (size_t i = 0; i < n; ++i) + tmp[i] = f32_to_bf16_u16(h[i]); __nv_bfloat16* d = nullptr; cudaMalloc(&d, n * sizeof(__nv_bfloat16)); cudaMemcpy(d, tmp.data(), n * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice); @@ -493,7 +525,8 @@ static int8_t* pack_and_upload_fused(const std::vector& wptrs, std::vector& out_scales, std::vector& out_ptrs) { int64_t N_total = 0; - for (int64_t n : Ns) N_total += n; + for (int64_t n : Ns) + N_total += n; std::vector stacked(N_total * K); out_scales.clear(); int64_t row_off = 0; @@ -514,21 +547,37 @@ static int8_t* pack_and_upload_fused(const std::vector& wptrs, BitvlaModelArch::~BitvlaModelArch() { #ifdef VLA_BITVLA_CUDA_KERNELS - if (lm_cuda_ctx) bitvla_lm_cuda_free(lm_cuda_ctx); - if (vit_cuda_ctx) bitvla_vit_cuda_free(vit_cuda_ctx); - if (fp32head_cuda_ctx) bitvla_fp32head_cuda_free(fp32head_cuda_ctx); - for (void* p : cuda_devptrs) if (p) cudaFree(p); - if (d_inputs_embeds) cudaFree(d_inputs_embeds); - if (d_last_hidden) cudaFree(d_last_hidden); - if (d_action_hidden) cudaFree(d_action_hidden); - if (d_action_ids) cudaFree(d_action_ids); - if (d_vit_patches) cudaFree(d_vit_patches); - if (d_vit_img_embeds) cudaFree(d_vit_img_embeds); - for (void* p : cpu_kept_ptrs) if (p) std::free(p); + if (lm_cuda_ctx) + bitvla_lm_cuda_free(lm_cuda_ctx); + if (vit_cuda_ctx) + bitvla_vit_cuda_free(vit_cuda_ctx); + if (fp32head_cuda_ctx) + bitvla_fp32head_cuda_free(fp32head_cuda_ctx); + for (void* p : cuda_devptrs) + if (p) + cudaFree(p); + if (d_inputs_embeds) + cudaFree(d_inputs_embeds); + if (d_last_hidden) + cudaFree(d_last_hidden); + if (d_action_hidden) + cudaFree(d_action_hidden); + if (d_action_ids) + cudaFree(d_action_ids); + if (d_vit_patches) + cudaFree(d_vit_patches); + if (d_vit_img_embeds) + cudaFree(d_vit_img_embeds); + for (void* p : cpu_kept_ptrs) + if (p) + std::free(p); #endif - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr bitvla_create(const std::string& mmproj_path, @@ -543,16 +592,19 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_F32); gguf_reader g("bitvla"); - if (!g.open(ckpt_path)) return nullptr; + if (!g.open(ckpt_path)) + return nullptr; if (!g.has("bitvla.architecture") && !g.has("general.architecture")) { std::fprintf(stderr, "vla(bitvla): %s is not a bitvla GGUF\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!load_config(g, *m, m->cfg)) + return nullptr; // Keep one reader open for the per-step token-embedding fetches (token_embd // stays on disk under int2 packing) and cache the constant stop-token row, // so predict() no longer re-opens and re-parses the GGUF twice per call. - if (!m->emb_reader.open(ckpt_path)) return nullptr; + if (!m->emb_reader.open(ckpt_path)) + return nullptr; m->stop_embed.resize((size_t) m->lm_hidden); { std::vector sid{ m->stop_id }; @@ -570,13 +622,19 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, // Not backend_init: the ggml graph stays on CPU and the LM offloads through // the ternary CUDA kernels below. m->backend = ggml_backend_cpu_init(); - if (!m->backend) { std::fprintf(stderr, "vla(bitvla): ggml_backend_cpu_init failed\n"); return nullptr; } + if (!m->backend) { + std::fprintf(stderr, "vla(bitvla): ggml_backend_cpu_init failed\n"); + return nullptr; + } ggml_backend_cpu_set_n_threads(m->backend, m->n_threads); std::printf("vla(bitvla): ggml backend = CPU (%d threads) - CUDA LM module activates below if available\n", m->n_threads); ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(bitvla): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(bitvla): ggml_init(ctx_weights) failed\n"); + return nullptr; + } WeightLoader L("bitvla", g, m->ctx_weights, m->matmul_type); auto mk_mm = [&](const char * name) { return L.gemm("%s", name); }; auto mk_f32 = [&](const char * name) { return L.f32 ("%s", name); }; @@ -645,9 +703,13 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->ah_b0_ln_w&&m->ah_b0_ln_b&&m->ah_b0_w&&m->ah_b0_b&& m->ah_b1_ln_w&&m->ah_b1_ln_b&&m->ah_b1_w&&m->ah_b1_b&& m->ah_ln2_w&&m->ah_ln2_b&&m->ah_fc2_w&&m->ah_fc2_b; - if (!ok) { std::fprintf(stderr, "vla(bitvla): weight tensor setup failed\n"); return nullptr; } + if (!ok) { + std::fprintf(stderr, "vla(bitvla): weight tensor setup failed\n"); + return nullptr; + } - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(bitvla): weights resident in %.2f GiB (%s); image_id=%d proprio_id=%d action_begin_id=%d stop_id=%d\n", ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), @@ -710,11 +772,27 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, const int64_t hq_dim = m->lm_q * m->lm_head_dim; const int64_t hkv_dim = m->lm_kv * m->lm_head_dim; - { auto r = load_bit(m->lm[L].Wq, hq_dim, m->lm_hidden); lyr.q_packed = r.first; lyr.q_ws = r.second; } - { auto r = load_bit(m->lm[L].Wk, hkv_dim, m->lm_hidden); lyr.k_packed = r.first; lyr.k_ws = r.second; } - { auto r = load_bit(m->lm[L].Wv, hkv_dim, m->lm_hidden); lyr.v_packed = r.first; lyr.v_ws = r.second; } + { + auto r = load_bit(m->lm[L].Wq, hq_dim, m->lm_hidden); + lyr.q_packed = r.first; + lyr.q_ws = r.second; + } + { + auto r = load_bit(m->lm[L].Wk, hkv_dim, m->lm_hidden); + lyr.k_packed = r.first; + lyr.k_ws = r.second; + } + { + auto r = load_bit(m->lm[L].Wv, hkv_dim, m->lm_hidden); + lyr.v_packed = r.first; + lyr.v_ws = r.second; + } - { auto r = load_bit(m->lm[L].Wo, m->lm_hidden, m->lm_hidden); lyr.o_packed = r.first; lyr.o_ws = r.second; } + { + auto r = load_bit(m->lm[L].Wo, m->lm_hidden, m->lm_hidden); + lyr.o_packed = r.first; + lyr.o_ws = r.second; + } if (m->packed_int2) { lyr.gate_up_packed = upload_int8((const uint8_t*) m->lm[L].Wgate_up->data, ggml_nbytes(m->lm[L].Wgate_up), m->cuda_devptrs); @@ -735,7 +813,11 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, lyr.gate_up_ws = upload_f32_scales(ws2.data(), 2, m->cuda_devptrs); } - { auto r = load_bit(m->lm[L].Wdown, m->lm_hidden, m->lm_inter); lyr.down_packed = r.first; lyr.down_ws = r.second; } + { + auto r = load_bit(m->lm[L].Wdown, m->lm_hidden, m->lm_inter); + lyr.down_packed = r.first; + lyr.down_ws = r.second; + } bitvla_lm_cuda_set_layer(m->lm_cuda_ctx, (int) L, &lyr); } if (!scales_ok) { @@ -746,9 +828,12 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, bitvla_lm_cuda_set_output_norm(m->lm_cuda_ctx, onorm); cudaError_t lm_ce = cudaMalloc(&m->d_inputs_embeds, (size_t) max_seq * m->lm_hidden * sizeof(__nv_bfloat16)); - if (lm_ce == cudaSuccess) lm_ce = cudaMalloc(&m->d_last_hidden, (size_t) max_seq * m->lm_hidden * sizeof(__nv_bfloat16)); - if (lm_ce == cudaSuccess) lm_ce = cudaMalloc(&m->d_action_hidden, (size_t) (m->num_actions_chunk * m->action_dim) * m->lm_hidden * sizeof(__nv_bfloat16)); - if (lm_ce == cudaSuccess) lm_ce = cudaMalloc(&m->d_action_ids, (size_t) (m->num_actions_chunk * m->action_dim) * sizeof(int32_t)); + if (lm_ce == cudaSuccess) + lm_ce = cudaMalloc(&m->d_last_hidden, (size_t) max_seq * m->lm_hidden * sizeof(__nv_bfloat16)); + if (lm_ce == cudaSuccess) + lm_ce = cudaMalloc(&m->d_action_hidden, (size_t) (m->num_actions_chunk * m->action_dim) * m->lm_hidden * sizeof(__nv_bfloat16)); + if (lm_ce == cudaSuccess) + lm_ce = cudaMalloc(&m->d_action_ids, (size_t) (m->num_actions_chunk * m->action_dim) * sizeof(int32_t)); // only enable the CUDA LM once every work buffer is really allocated. if (lm_ce != cudaSuccess) { std::fprintf(stderr, "vla(bitvla): CUDA LM buffer alloc failed (%s); using CPU LM\n", @@ -785,20 +870,44 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, vl.ln2_w = upload_bf16_from_f32((const float*) m->vit[L].ln2w->data, m->vit_hidden, m->cuda_devptrs); vl.ln2_b = upload_bf16_from_f32((const float*) m->vit[L].ln2b->data, m->vit_hidden, m->cuda_devptrs); - { auto r = load_bit(m->vit[L].Wq, m->vit_hidden, m->vit_hidden); vl.q_packed = r.first; vl.q_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wq, m->vit_hidden, m->vit_hidden); + vl.q_packed = r.first; + vl.q_ws = r.second; + } vl.q_b = upload_bf16_from_f32((const float*) m->vit[L].bq->data, m->vit_hidden, m->cuda_devptrs); - { auto r = load_bit(m->vit[L].Wk, m->vit_hidden, m->vit_hidden); vl.k_packed = r.first; vl.k_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wk, m->vit_hidden, m->vit_hidden); + vl.k_packed = r.first; + vl.k_ws = r.second; + } vl.k_b = upload_bf16_from_f32((const float*) m->vit[L].bk->data, m->vit_hidden, m->cuda_devptrs); - { auto r = load_bit(m->vit[L].Wv, m->vit_hidden, m->vit_hidden); vl.v_packed = r.first; vl.v_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wv, m->vit_hidden, m->vit_hidden); + vl.v_packed = r.first; + vl.v_ws = r.second; + } vl.v_b = upload_bf16_from_f32((const float*) m->vit[L].bv->data, m->vit_hidden, m->cuda_devptrs); - { auto r = load_bit(m->vit[L].Wo, m->vit_hidden, m->vit_hidden); vl.o_packed = r.first; vl.o_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wo, m->vit_hidden, m->vit_hidden); + vl.o_packed = r.first; + vl.o_ws = r.second; + } vl.o_b = upload_bf16_from_f32((const float*) m->vit[L].bo->data, m->vit_hidden, m->cuda_devptrs); - { auto r = load_bit(m->vit[L].Wfc1, m->vit_inter, m->vit_hidden); vl.fc1_packed = r.first; vl.fc1_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wfc1, m->vit_inter, m->vit_hidden); + vl.fc1_packed = r.first; + vl.fc1_ws = r.second; + } vl.fc1_b = upload_bf16_from_f32((const float*) m->vit[L].bfc1->data, m->vit_inter, m->cuda_devptrs); if (m->packed_int2) { - { auto r = load_bit(m->vit[L].Wfc2, m->vit_hidden, ffn_pad); vl.fc2_packed = r.first; vl.fc2_ws = r.second; } + { + auto r = load_bit(m->vit[L].Wfc2, m->vit_hidden, ffn_pad); + vl.fc2_packed = r.first; + vl.fc2_ws = r.second; + } vl.fc2_b = upload_bf16_from_f32((const float*) m->vit[L].bfc2->data, m->vit_hidden, m->cuda_devptrs); } else { const float* W = (const float*) m->vit[L].Wfc2->data; @@ -915,7 +1024,8 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->ah_ln2_w, m->ah_ln2_b, m->ah_fc2_w, m->ah_fc2_b, }; for (ggml_tensor* t : keep_tensors) { - if (!t) continue; + if (!t) + continue; const size_t nb = ggml_nbytes(t); void* copy = std::malloc(nb); if (!copy) { @@ -957,18 +1067,22 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { const char* _dump_dir = std::getenv("VLA_BITVLA_DUMP_DIR"); auto _dump_bin = [&](const char* name, const float* data, size_t nelem) { - if (!_dump_dir) return; + if (!_dump_dir) + return; std::string path = std::string(_dump_dir) + "/" + name + ".bin"; FILE* f = std::fopen(path.c_str(), "wb"); - if (!f) return; + if (!f) + return; std::fwrite(data, sizeof(float), nelem, f); std::fclose(f); }; auto _dump_manifest = [&](const std::string& line) { - if (!_dump_dir) return; + if (!_dump_dir) + return; std::string path = std::string(_dump_dir) + "/manifest.txt"; FILE* f = std::fopen(path.c_str(), "a"); - if (!f) return; + if (!f) + return; std::fwrite(line.data(), 1, line.size(), f); std::fputc('\n', f); std::fclose(f); @@ -1032,7 +1146,8 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { if (cuda_vit_ready) { std::vector patches_bf16((size_t) N * patch_flat); - for (size_t i = 0; i < patches_bf16.size(); ++i) patches_bf16[i] = f32_to_bf16_u16(patches[i]); + for (size_t i = 0; i < patches_bf16.size(); ++i) + patches_bf16[i] = f32_to_bf16_u16(patches[i]); cudaMemcpy(d_vit_patches, patches_bf16.data(), patches_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice); int rc = bitvla_vit_cuda_forward(vit_cuda_ctx, d_vit_patches, d_vit_img_embeds, 0); if (rc != 0) { std::fprintf(stderr, "vla(bitvla): CUDA ViT forward failed (view %lld)\n", (long long) v); return {}; } @@ -1087,7 +1202,8 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::vector proprio_embed_host((size_t) hidden_l); // Like the other archs: a caller may leave the proprio vector out. std::vector state_host((size_t) proprio_dim, 0.0f); - if (in.state) std::memcpy(state_host.data(), in.state, (size_t) proprio_dim * sizeof(float)); + if (in.state) + std::memcpy(state_host.data(), in.state, (size_t) proprio_dim * sizeof(float)); #ifdef VLA_BITVLA_CUDA_KERNELS if (cuda_fp32head_ready) { if (bitvla_fp32head_proprio_forward(fp32head_cuda_ctx, state_host.data(), proprio_embed_host.data(), 0) != 0) { @@ -1121,7 +1237,8 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { int64_t n_image_markers = 0, n_proprio_markers = 0; for (int64_t i = 0; i < n_lang_in; ++i) { - if (in.lang_tokens[i] == image_token_id) n_image_markers++; + if (in.lang_tokens[i] == image_token_id) + n_image_markers++; else if (in.lang_tokens[i] == proprio_pad_id) n_proprio_markers++; } const bool full_prefix = (n_image_markers == n_img_tok && n_proprio_markers == 1); @@ -1209,14 +1326,16 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { if (cuda_lm_ready && seq <= cuda_max_seq) { std::vector in_bf16((size_t) seq * hidden_l); - for (size_t i = 0; i < in_bf16.size(); ++i) in_bf16[i] = f32_to_bf16_u16(inputs_embeds[i]); + for (size_t i = 0; i < in_bf16.size(); ++i) + in_bf16[i] = f32_to_bf16_u16(inputs_embeds[i]); cudaMemcpy(d_inputs_embeds, in_bf16.data(), in_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice); int rc = bitvla_lm_cuda_forward(lm_cuda_ctx, d_inputs_embeds, d_last_hidden, (int) seq, 0); if (rc != 0) { std::fprintf(stderr, "vla(bitvla): CUDA LM forward failed\n"); return {}; } std::vector aids(n_action); - for (int64_t i = 0; i < n_action; ++i) aids[i] = (int32_t) (seq - 2 - n_action + i); + for (int64_t i = 0; i < n_action; ++i) + aids[i] = (int32_t) (seq - 2 - n_action + i); cudaMemcpy(d_action_ids, aids.data(), n_action * sizeof(int32_t), cudaMemcpyHostToDevice); bitvla_gather_rows_bf16(d_last_hidden, d_action_hidden, d_action_ids, (int) n_action, (int) hidden_l, 0); @@ -1254,16 +1373,19 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(x_in, inputs_embeds.data(), 0, ggml_nbytes(x_in)); std::vector pos_v(seq); - for (int64_t i = 0; i < seq; ++i) pos_v[i] = (int32_t) i; + for (int64_t i = 0; i < seq; ++i) + pos_v[i] = (int32_t) i; ggml_backend_tensor_set(positions, pos_v.data(), 0, ggml_nbytes(positions)); std::vector aids(n_action); - for (int64_t i = 0; i < n_action; ++i) aids[i] = (int32_t) (seq - 2 - n_action + i); + for (int64_t i = 0; i < n_action; ++i) + aids[i] = (int32_t) (seq - 2 - n_action + i); ggml_backend_tensor_set(action_ids, aids.data(), 0, ggml_nbytes(action_ids)); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): lm prefill compute failed\n"); return {}; } ggml_backend_tensor_get(action_hidden, last_hidden_at_actions.data(), 0, (size_t) n_action * hidden_l * sizeof(float)); } - if (timing_phase) stats.ms_prefill = std::chrono::duration(clk::now() - t_p0).count(); + if (timing_phase) + stats.ms_prefill = std::chrono::duration(clk::now() - t_p0).count(); _dump_bin("ah_input", last_hidden_at_actions.data(), last_hidden_at_actions.size()); _dump_manifest(std::string("ah_input fp32 1 ") + std::to_string(num_actions_chunk) + @@ -1311,7 +1433,8 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): action_head compute failed\n"); return {}; } ggml_backend_tensor_get(y, normalized_actions.data(), 0, (size_t) chunk * action_dim * sizeof(float)); } - if (timing_phase) stats.ms_denoise = std::chrono::duration(clk::now() - t_d0).count(); + if (timing_phase) + stats.ms_denoise = std::chrono::duration(clk::now() - t_d0).count(); _dump_bin("ah_norm_actions", normalized_actions.data(), normalized_actions.size()); _dump_manifest(std::string("ah_norm_actions fp32 1 ") + std::to_string(num_actions_chunk) + diff --git a/src/models/dit_common.h b/src/models/dit_common.h index 83f6cc7..c293180 100644 --- a/src/models/dit_common.h +++ b/src/models/dit_common.h @@ -48,14 +48,22 @@ inline ggml_tensor * adaln(ggml_context * C, ggml_tensor * x, ggml_tensor * temb inline void timesteps_proj(int64_t bucket, std::vector & out) { const int64_t half = 128; const float lm = std::log(10000.0f); const float t = (float) bucket; out.assign(256, 0.0f); - for (int64_t i = 0; i < half; ++i) { const float emb = t * std::exp(-lm * (float) i / (float) (half - 1)); out[i] = std::cos(emb); out[half + i] = std::sin(emb); } + for (int64_t i = 0; i < half; ++i) { + const float emb = t * std::exp(-lm * (float) i / (float) (half - 1)); + out[i] = std::cos(emb); + out[half + i] = std::sin(emb); + } } // Broadcast across the horizon. sin first, then cos. inline void action_sinusoid(int64_t bucket, int64_t dim, int64_t T, std::vector & out) { const int64_t half = dim / 2; const float step = std::log(10000.0f) / (float) half; const float t = (float) bucket; out.assign((size_t) T * dim, 0.0f); - for (int64_t tk = 0; tk < T; ++tk) for (int64_t i = 0; i < half; ++i) { const float emb = t * std::exp(-(float) i * step); out[tk * dim + i] = std::sin(emb); out[tk * dim + half + i] = std::cos(emb); } + for (int64_t tk = 0; tk < T; ++tk) for (int64_t i = 0; i < half; ++i) { + const float emb = t * std::exp(-(float) i * step); + out[tk * dim + i] = std::sin(emb); + out[tk * dim + half + i] = std::cos(emb); + } } // Log-spaced periods rather than frequencies, the openpi convention shared by @@ -78,7 +86,8 @@ inline void build_causal_mask(int64_t seq, std::vector & out) { out.assign((size_t) seq * seq, 0.0f); const float NEG = -std::numeric_limits::infinity(); for (int64_t q = 0; q < seq; ++q) - for (int64_t kv = q + 1; kv < seq; ++kv) out[q * seq + kv] = NEG; + for (int64_t kv = q + 1; kv < seq; ++kv) + out[q * seq + kv] = NEG; } } // namespace vla diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 269b4cd..8be91c3 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -73,7 +73,9 @@ struct Evo1ModelArch : public ModelArchBase { struct MainKey { int64_t seq=-1, nsteps=-1; - bool operator==(const MainKey & o) const { return seq==o.seq && nsteps==o.nsteps; } + bool operator==(const MainKey & o) const { + return seq==o.seq && nsteps==o.nsteps; + } }; struct MainIO { ggml_tensor *t_embeds=nullptr,*t_pos=nullptr,*t_lmmask=nullptr,*t_qmask=nullptr; @@ -149,7 +151,8 @@ ggml_tensor * build_qwen2_layer(ggml_context * C, const Evo1ModelArch & m, const ggml_tensor * kqv = ggml_mul_mat(C, V, aw); ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, kqv, 0, 2, 1, 3)), hq, seq); ggml_tensor * attn_out = mm_act(C, w.Wo, as_type(C, att, at), at); - if (qmask) attn_out = ggml_mul(C, attn_out, qmask); + if (qmask) + attn_out = ggml_mul(C, attn_out, qmask); ggml_tensor * h_attn = ggml_add(C, h, attn_out); ggml_tensor * h_n2 = ggml_mul(C, ggml_rms_norm(C, h_attn, m.lm_rms_eps), w.ffn_norm); ggml_tensor * gate = ggml_silu(C, mm_act(C, w.Wgate, h_n2, at)); @@ -260,7 +263,8 @@ ggml_tensor * build_internvit_view(ggml_context * C, const Evo1ModelArch & m, gg // the activation dtype from here ggml_tensor * x = as_type(C, ggml_add(C, ggml_concat(C, cls2d, patches, 1), m.vit_pos), m.act_type); - for (int64_t i = 0; i < m.vit_layers; ++i) x = build_internvit_layer(C, m, m.vit[i], x, n_tok); + for (int64_t i = 0; i < m.vit_layers; ++i) + x = build_internvit_layer(C, m, m.vit[i], x, n_tok); ggml_tensor * pnc = ggml_cont(C, ggml_view_2d(C, x, H, n_patches, x->nb[1], x->nb[1])); ggml_tensor * s1 = ggml_reshape_3d(C, pnc, 2 * H, sgrid, grid); @@ -294,11 +298,16 @@ bool load_config(const gguf_reader & g, Evo1ModelArch & m, Config & cfg) { u("real_state_dim", m.real_state_dim); u("real_action_dim", m.real_action_dim); u("vit_hidden", m.vit_hidden); u("vit_layers", m.vit_layers); u("vit_heads", m.vit_heads); u("vit_inter", m.vit_inter); u("image_size", m.image_size); u("patch_size", m.patch_size); - if (g.has("evo1.lm_rms_eps")) m.lm_rms_eps = g.f32("evo1.lm_rms_eps"); - if (g.has("evo1.proj_ln_eps")) m.proj_ln_eps = g.f32("evo1.proj_ln_eps"); - if (g.has("evo1.norm_eps")) m.norm_eps_denom = g.f32("evo1.norm_eps"); - if (g.has("evo1.vit_ln_eps")) m.vit_ln_eps = g.f32("evo1.vit_ln_eps"); - if (g.has("evo1.lm_rope_theta")) m.lm_rope_base = (float) g.f64("evo1.lm_rope_theta"); + if (g.has("evo1.lm_rms_eps")) + m.lm_rms_eps = g.f32("evo1.lm_rms_eps"); + if (g.has("evo1.proj_ln_eps")) + m.proj_ln_eps = g.f32("evo1.proj_ln_eps"); + if (g.has("evo1.norm_eps")) + m.norm_eps_denom = g.f32("evo1.norm_eps"); + if (g.has("evo1.vit_ln_eps")) + m.vit_ln_eps = g.f32("evo1.vit_ln_eps"); + if (g.has("evo1.lm_rope_theta")) + m.lm_rope_base = (float) g.f64("evo1.lm_rope_theta"); if (m.embed_dim != m.lm_hidden) { std::fprintf(stderr, "vla(evo1): embed_dim (%lld) != lm_hidden (%lld) - not handled\n", (long long) m.embed_dim, (long long) m.lm_hidden); return false; @@ -339,9 +348,12 @@ bool load_config(const gguf_reader & g, Evo1ModelArch & m, Config & cfg) { } Evo1ModelArch::~Evo1ModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr evo1_create(const std::string& mmproj_path, @@ -356,12 +368,14 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->gguf_path = ckpt_path; m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); - if (!m->io.open(ckpt_path)) return nullptr; + if (!m->io.open(ckpt_path)) + return nullptr; gguf_reader & g = m->io; if (!g.has("evo1.architecture")) { std::fprintf(stderr, "vla(evo1): %s is not an evo1 GGUF (no evo1.architecture KV)\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!load_config(g, *m, m->cfg)) + return nullptr; std::printf("vla(evo1): lm=%lldd×%lldL (%lldq/%lldkv×%lld) inter=%lld embed=%lld dit=%lldL×%lldh " "horizon=%lld per_a=%lld N_steps=%lld resident matmul=%s\n", (long long) m->lm_hidden, (long long) m->lm_layers, (long long) m->n_q, (long long) m->n_kv, @@ -371,7 +385,9 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, { const Backend b = backend_init("vla(evo1)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. @@ -389,7 +405,10 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(evo1): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(evo1): ggml_init(ctx_weights) failed\n"); + return nullptr; + } // The vision tower is optional here, so misses are reported by the ok chain // below rather than by the loader. WeightLoader L("evo1", g, m->ctx_weights, m->matmul_type); @@ -465,9 +484,13 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, } else { std::printf("vla(evo1): note - no vit.*/mm.* weights in the GGUF; predict() will require Inputs::precomputed_img_emb\n"); } - if (!ok) { std::fprintf(stderr, "vla(evo1): weight tensor setup failed\n"); return nullptr; } + if (!ok) { + std::fprintf(stderr, "vla(evo1): weight tensor setup failed\n"); + return nullptr; + } - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(evo1): weights resident in %.2f GiB (%s)%s\n", ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), @@ -516,7 +539,10 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { // The branches are independent, so the arithmetic per view is unchanged // - only the submission pattern differs. const size_t want_arena = (size_t) 32 * 1024 * 1024 * (size_t) std::max(n_views, 1); - if (want_arena > vision_arena) { vision_scratch.release(); vision_arena = want_arena; } + if (want_arena > vision_arena) { + vision_scratch.release(); + vision_arena = want_arena; + } ggml_context * VC = vision_scratch.reset(vision_arena); if (!VC) { std::fprintf(stderr, "vla(evo1): ggml_init(vision ctx) failed\n"); return {}; } std::vector t_px((size_t) n_views), t_ie((size_t) n_views); @@ -527,7 +553,8 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { ggml_set_output(t_ie[v]); } ggml_cgraph * vg = ggml_new_graph_custom(VC, (size_t) 8192 * std::max(n_views, 1), false); - for (int64_t v = 0; v < n_views; ++v) ggml_build_forward_expand(vg, t_ie[v]); + for (int64_t v = 0; v < n_views; ++v) + ggml_build_forward_expand(vg, t_ie[v]); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(evo1): vision ggml_gallocr_alloc_graph failed\n"); return {}; @@ -559,18 +586,24 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { bool pre_built = false; for (int j = 0; j < in.n_lang; ++j) - if (in.lang_tokens[j] == (int32_t) img_ctx_id) { pre_built = true; break; } + if (in.lang_tokens[j] == (int32_t) img_ctx_id) { + pre_built = true; + break; + } std::vector input_ids; input_ids.reserve(max_text_length); if (pre_built) { - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); + for (int j = 0; j < in.n_lang; ++j) + input_ids.push_back(in.lang_tokens[j]); } else { for (int64_t v = 0; v < n_views; ++v) { input_ids.push_back((int32_t) img_start_id); - for (int64_t k = 0; k < num_image_token; ++k) input_ids.push_back((int32_t) img_ctx_id); + for (int64_t k = 0; k < num_image_token; ++k) + input_ids.push_back((int32_t) img_ctx_id); input_ids.push_back((int32_t) img_end_id); } - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); + for (int j = 0; j < in.n_lang; ++j) + input_ids.push_back(in.lang_tokens[j]); } const int64_t n_real = (int64_t) input_ids.size(); if (n_real > max_text_length) { @@ -606,9 +639,11 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { in.attention_mask_n, (long long) SEQ); return {}; } - for (int64_t p = 0; p < SEQ; ++p) attn_ok[p] = in.attention_mask[p] ? 1 : 0; + for (int64_t p = 0; p < SEQ; ++p) + attn_ok[p] = in.attention_mask[p] ? 1 : 0; } else { - for (int64_t p = 0; p < n_real; ++p) attn_ok[p] = 1; + for (int64_t p = 0; p < n_real; ++p) + attn_ok[p] = 1; } std::vector state_norm(per_a, 0.0f); @@ -616,11 +651,16 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { const float lo = state_min[i], hi = state_max[i]; // The converter zero-pads stats past real_state_dim, so lo == hi == 0 there // and the affine below would map anything to -1. - if (hi <= lo) { state_norm[i] = 0.0f; continue; } + if (hi <= lo) { + state_norm[i] = 0.0f; + continue; + } const float sv = in.state ? in.state[i] : 0.0f; float xn = 2.0f * (sv - lo) / (hi - lo + norm_eps_denom) - 1.0f; - if (xn < -1.0f) xn = -1.0f; - if (xn > 1.0f) xn = 1.0f; + if (xn < -1.0f) + xn = -1.0f; + if (xn > 1.0f) + xn = 1.0f; state_norm[i] = xn; } @@ -656,7 +696,8 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { const ggml_type at = act_type; ggml_tensor * h = as_type(C, t_embeds, at); - for (int64_t i = 0; i < lm_layers; ++i) h = build_qwen2_layer(C, *this, lm[i], h, t_pos, t_lmmask, SEQ, t_qmask); + for (int64_t i = 0; i < lm_layers; ++i) + h = build_qwen2_layer(C, *this, lm[i], h, t_pos, t_lmmask, SEQ, t_qmask); ggml_tensor * context = ggml_mul(C, ggml_rms_norm(C, h, lm_rms_eps), lm_output_norm); ggml_tensor * se = ggml_relu(C, ggml_add(C, mm_act(C, state_W1, as_type(C, t_state, at), at), state_b1)); @@ -742,7 +783,12 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { ggml_tensor * t_amask = gio.t_amask, * x_action = gio.x_action; ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); - { std::vector pp(SEQ); for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } + { + std::vector pp(SEQ); + for (int64_t i = 0; i < SEQ; ++i) + pp[i] = (int32_t) i; + ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); + } { std::vector mk((size_t) SEQ * SEQ); const float NEG = -std::numeric_limits::infinity(); for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ + kv] = (kv <= q && attn_ok[kv]) ? 0.0f : NEG; ggml_backend_tensor_set(t_lmmask, mk.data(), 0, ggml_nbytes(t_lmmask)); } diff --git a/src/models/gr00tn1d5.cpp b/src/models/gr00tn1d5.cpp index 2d6292d..2bb0b1d 100644 --- a/src/models/gr00tn1d5.cpp +++ b/src/models/gr00tn1d5.cpp @@ -60,7 +60,9 @@ struct Gr00tN1d5ModelArch : public ModelArchBase { struct MainKey { int64_t seq=-1, nsteps=-1; - bool operator==(const MainKey & o) const { return seq==o.seq && nsteps==o.nsteps; } + bool operator==(const MainKey & o) const { + return seq==o.seq && nsteps==o.nsteps; + } }; struct MainIO { ggml_tensor *t_embeds=nullptr,*t_pos=nullptr,*t_lmmask=nullptr,*t_state=nullptr,*t_x0=nullptr,*actions=nullptr; @@ -134,7 +136,8 @@ bool load_config(const gguf_reader & g, Gr00tN1d5ModelArch & m, Config & cfg) { F(fk("norm_out_eps" ), m.dit.cfg.norm_out_eps); F(fk("vlln_eps" ), m.vlln_eps); - if (g.has(fk("lm_rope_theta"))) m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); + if (g.has(fk("lm_rope_theta"))) + m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); m.vit.enc.cfg.head_dim = m.vit.enc.cfg.hidden/m.vit.enc.cfg.heads; m.vlsa.cfg.hidden = m.bb_embed_dim; @@ -151,7 +154,8 @@ bool load_config(const gguf_reader & g, Gr00tN1d5ModelArch & m, Config & cfg) { const std::string js = g.str(fk("embodiment_tag_mapping")); const std::string key = std::string("\"")+e+"\":"; const size_t p = js.find(key); - if (p != std::string::npos) m.aex.embodiment_id = std::strtol(js.c_str()+p+key.size(), nullptr, 10); + if (p != std::string::npos) + m.aex.embodiment_id = std::strtol(js.c_str()+p+key.size(), nullptr, 10); else std::fprintf(stderr, "vla(gr00tn1d5): embodiment tag '%s' not in embodiment_tag_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); } } @@ -190,9 +194,12 @@ bool load_config(const gguf_reader & g, Gr00tN1d5ModelArch & m, Config & cfg) { } Gr00tN1d5ModelArch::~Gr00tN1d5ModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, @@ -206,13 +213,15 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); m->lm.cfg.rope.freq_base = 1000000.0f; - if (!m->io.open(ckpt_path)) return nullptr; + if (!m->io.open(ckpt_path)) + return nullptr; gguf_reader & g = m->io; if (!g.has("gr00t_n1_5.architecture")) { std::fprintf(stderr, "vla(gr00tn1d5): %s is not a gr00t_n1_5 GGUF\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!load_config(g, *m, m->cfg)) + return nullptr; std::printf("vla(gr00tn1d5): vit=%lldd×%lldL×%lldh n_img_tok=%lld lm=Qwen3 %lldd×%lldL (%lldq/%lldkv×%lld) " "dit=%lldL×%lldh×%lld(inner %lld) interleave=%lld vlsa=%lldL×%lldh×%lld in_emb=%lld horizon=%lld action_dim=%lld N_steps=%lld embodiment=%lld resident=%s\n", @@ -224,12 +233,16 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); const Backend b = backend_init("vla(gr00tn1d5)", m->n_threads); - if (!b.handle) return nullptr; + if (!b.handle) + return nullptr; m->backend = b.handle; ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d5): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(gr00tn1d5): ggml_init(ctx_weights) failed\n"); + return nullptr; + } WeightLoader L("gr00tn1d5", g, m->ctx_weights, m->matmul_type); @@ -247,7 +260,8 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, m->future_tokens = L.f32("aex.future_tokens"); m->dit.declare(L, "aex.dit"); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(gr00tn1d5): weights resident in %.2f GiB (%s) - incl. SigLIP vision tower; embodiment id %lld\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), @@ -344,7 +358,8 @@ std::vector Gr00tN1d5ModelArch::predict(const Inputs& in) { std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); for (int64_t i = 0; i < dit.cfg.layers; ++i) { - if (dit_interleave && (i%2 == 1)) continue; + if (dit_interleave && (i%2 == 1)) + continue; dit.kv(C, dit.blk[i], vl_embs, &Kc[i], &Vc[i]); } @@ -381,7 +396,8 @@ std::vector Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); std::vector pp(SEQ); - for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; + for (int64_t i = 0; i < SEQ; ++i) + pp[i] = (int32_t) i; ggml_backend_tensor_set(gio.t_pos, pp.data(), 0, ggml_nbytes(gio.t_pos)); std::vector mask; @@ -389,7 +405,8 @@ std::vector Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); std::vector st(max_state_dim, 0.0f); - for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; + for (int64_t i = 0; i < max_state_dim; ++i) + st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(gio.t_state, st.data(), 0, ggml_nbytes(gio.t_state)); ggml_backend_tensor_set(gio.t_x0, x_init.data(), 0, ggml_nbytes(gio.t_x0)); diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index ebbac5f..ec7816c 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -139,7 +139,8 @@ bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { F(fk("vlln_eps" ), m.vlln_eps); F(fk("connector_ln_eps" ), m.connector_ln_eps); - if (g.has(fk("lm_rope_theta"))) m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); + if (g.has(fk("lm_rope_theta"))) + m.lm.cfg.rope.freq_base = (float) g.f64(fk("lm_rope_theta")); m.vit.enc.cfg.head_dim = m.vit.enc.cfg.hidden/m.vit.enc.cfg.heads; m.lm.cfg.rope.n_dims = (int) m.lm.cfg.head_dim; @@ -150,14 +151,17 @@ bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { auto lookup = [&](const char * key) -> long { const std::string k = std::string("\"")+key+"\""; size_t p = js.find(k); - if (p == std::string::npos) return -1; + if (p == std::string::npos) + return -1; p = js.find(':', p+k.size()); - if (p == std::string::npos) return -1; + if (p == std::string::npos) + return -1; return std::strtol(js.c_str()+p+1, nullptr, 10); }; const long gr1 = lookup("gr1"); - if (gr1 >= 0) m.aex.embodiment_id = gr1; + if (gr1 >= 0) + m.aex.embodiment_id = gr1; if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { char * end = nullptr; @@ -166,7 +170,8 @@ bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { m.aex.embodiment_id = v; } else { const long id = lookup(e); - if (id >= 0) m.aex.embodiment_id = id; + if (id >= 0) + m.aex.embodiment_id = id; else std::fprintf(stderr, "vla(gr00tn1d6): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); } } @@ -223,9 +228,12 @@ bool load_config(const gguf_reader & g, Gr00tN1d6ModelArch & m, Config & cfg) { } Gr00tN1d6ModelArch::~Gr00tN1d6ModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, @@ -239,13 +247,15 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); m->lm.cfg.rope.freq_base = 1000000.0f; - if (!m->io.open(ckpt_path)) return nullptr; + if (!m->io.open(ckpt_path)) + return nullptr; gguf_reader & g = m->io; if (!g.has("gr00t_n1_6.architecture")) { std::fprintf(stderr, "vla(gr00tn1d6): %s is not a gr00t_n1_6 GGUF\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!load_config(g, *m, m->cfg)) + return nullptr; std::printf("vla(gr00tn1d6): vit=%lldd×%lldL×%lldh (Linear patch embed) pixel_shuffle÷%lld ⇒ n_img_tok=%lld mlp1=LN(%lld)→Linear→GELU→Linear " "lm=Qwen3 %lldd×%lldL (%lldq/%lldkv×%lld) dit=AlternateVLDiT %lldL×%lldh×%lld(inner %lld) attend_text_every_n=%lld in_emb=%lld " @@ -257,12 +267,16 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"); const Backend b = backend_init("vla(gr00tn1d6)", m->n_threads); - if (!b.handle) return nullptr; + if (!b.handle) + return nullptr; m->backend = b.handle; ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d6): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(gr00tn1d6): ggml_init(ctx_weights) failed\n"); + return nullptr; + } WeightLoader L("gr00tn1d6", g, m->ctx_weights, m->matmul_type); @@ -283,7 +297,8 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, m->aex.declare(L, "aex"); m->dit.declare(L, "aex.dit"); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(gr00tn1d6): weights resident in %.2f GiB (%s) - incl. SigLIP2 vision tower; embodiment id %lld\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), @@ -351,7 +366,10 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { bool vok = true; for (int64_t v = 0; v < n_views && vok; ++v) { - if (!preprocess_image_patches("gr00tn1d6", in.images[v], image_size, patch_size, patches)) { vok = false; break; } + if (!preprocess_image_patches("gr00tn1d6", in.images[v], image_size, patch_size, patches)) { + vok = false; + break; + } std::memcpy(patches_all.data()+(size_t) v*patch_dim*n_patches, patches.data(), patches.size()*sizeof(float)); } if (vok) { @@ -373,7 +391,8 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { vok = false; } } - if (vok) ggml_backend_tensor_get(vit_embeds, img_emb_host.data(), 0, ggml_nbytes(vit_embeds)); + if (vok) + ggml_backend_tensor_get(vit_embeds, img_emb_host.data(), 0, ggml_nbytes(vit_embeds)); stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); if (!vok) return {}; img_emb_ptr = img_emb_host.data(); @@ -406,7 +425,8 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_tensor * t_img_idx = ggml_new_tensor_1d(C, GGML_TYPE_I32, n_img); ggml_set_input(t_img_idx); ggml_tensor * t_txt_idx = (SEQ_TXT > 0) ? ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ_TXT) : nullptr; - if (t_txt_idx) ggml_set_input(t_txt_idx); + if (t_txt_idx) + ggml_set_input(t_txt_idx); std::vector t_tau(num_steps), t_tproj(num_steps); for (int64_t s = 0; s < num_steps; ++s) { @@ -426,7 +446,8 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); for (int64_t i = 0; i < dit.cfg.layers; ++i) { - if (dit_interleave && (i%2 == 1)) continue; + if (dit_interleave && (i%2 == 1)) + continue; dit.kv(C, dit.blk[i], (i%every2 == 0) ? vl_txt : vl_img, &Kc[i], &Vc[i]); } @@ -438,9 +459,11 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < dit.cfg.layers; ++i) { ggml_tensor * enc; - if (dit_interleave && (i%2 == 1)) enc = nullptr; + if (dit_interleave && (i%2 == 1)) + enc = nullptr; else if (i%every2 == 0) enc = vl_txt; - else enc = vl_img; + else + enc = vl_img; hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); } @@ -465,7 +488,8 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); std::vector pp(SEQ); - for (int64_t i = 0; i < SEQ; ++i) pp[i] = (int32_t) i; + for (int64_t i = 0; i < SEQ; ++i) + pp[i] = (int32_t) i; ggml_backend_tensor_set(gio.t_pos, pp.data(), 0, ggml_nbytes(gio.t_pos)); std::vector mask; @@ -473,12 +497,14 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); std::vector st(max_state_dim, 0.0f); - for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; + for (int64_t i = 0; i < max_state_dim; ++i) + st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(gio.t_state, st.data(), 0, ggml_nbytes(gio.t_state)); ggml_backend_tensor_set(gio.t_x0, x_init.data(), 0, ggml_nbytes(gio.t_x0)); ggml_backend_tensor_set(gio.t_img_idx, prompt.image_pos.data(), 0, ggml_nbytes(gio.t_img_idx)); - if (gio.t_txt_idx) ggml_backend_tensor_set(gio.t_txt_idx, prompt.text_pos.data(), 0, ggml_nbytes(gio.t_txt_idx)); + if (gio.t_txt_idx) + ggml_backend_tensor_set(gio.t_txt_idx, prompt.text_pos.data(), 0, ggml_nbytes(gio.t_txt_idx)); for (int64_t s = 0; s < num_steps; ++s) { const int64_t bucket = (int64_t) ((double) s/(double) num_steps*(double) num_buckets); diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 7c0d996..4ae4c2b 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -155,11 +155,15 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { if (const char * ns = std::getenv("VLA_NUM_STEPS")) { char * end = nullptr; long v = std::strtol(ns, &end, 10); - if (end && *end == '\0' && v >= 1) { m.num_steps = (int64_t) v; std::fprintf(stderr, "vla(gr00tn1d7): VLA_NUM_STEPS override → num_steps=%lld\n", (long long) v); } + if (end && *end == '\0' && v >= 1) { + m.num_steps = (int64_t) v; + std::fprintf(stderr, "vla(gr00tn1d7): VLA_NUM_STEPS override → num_steps=%lld\n", (long long) v); + } } F(fk("vit_ln_eps"), m.vit_ln_eps); F(fk("lm_rms_eps"), m.lm_rms_eps); F(fk("ln_eps"), m.ln_eps); F(fk("norm_out_eps"), m.norm_out_eps); F(fk("vlln_eps"), m.vlln_eps); F(fk("vlsa_ln_eps"), m.vlsa_ln_eps); F(fk("connector_ln_eps"), m.connector_ln_eps); F(fk("vit_rope_theta"), m.vit_rope_base); - if (g.has(fk("lm_rope_theta"))) m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); + if (g.has(fk("lm_rope_theta"))) + m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); m.lm.cfg.hidden = m.lm_hidden; m.lm.cfg.layers = m.lm_layers; @@ -202,11 +206,15 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { long ls = lookup("libero_sim"); if (ls >= 0) m.aex.embodiment_id = ls; if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { char * end = nullptr; long v = std::strtol(e, &end, 10); - if (end && *end == '\0') m.aex.embodiment_id = v; + if (end && *end == '\0') + m.aex.embodiment_id = v; else { long id = lookup(e); if (id >= 0) m.aex.embodiment_id = id; else std::fprintf(stderr, "vla(gr00tn1d7): embodiment tag '%s' not in embodiment_id_mapping; using id %lld\n", e, (long long) m.aex.embodiment_id); } } } - if (m.aex.embodiment_id < 0 || m.aex.embodiment_id >= m.max_embodiments) { std::fprintf(stderr, "vla(gr00tn1d7): embodiment id %lld out of range [0,%lld)\n", (long long) m.aex.embodiment_id, (long long) m.max_embodiments); return false; } + if (m.aex.embodiment_id < 0 || m.aex.embodiment_id >= m.max_embodiments) { + std::fprintf(stderr, "vla(gr00tn1d7): embodiment id %lld out of range [0,%lld)\n", (long long) m.aex.embodiment_id, (long long) m.max_embodiments); + return false; + } cfg = Config{}; cfg.n_img = 64; cfg.n_lang = m.max_seq_len; cfg.n_state = 1; @@ -226,9 +234,12 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { Gr00tN1d7ModelArch::~Gr00tN1d7ModelArch() { mg.release(); - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, @@ -243,9 +254,14 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("gr00tn1d7"); - if (!g.open(ckpt_path)) return nullptr; - if (!g.has("gr00t_n1_7.architecture")) { std::fprintf(stderr, "vla(gr00tn1d7): %s is not a gr00t_n1_7 GGUF\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!g.open(ckpt_path)) + return nullptr; + if (!g.has("gr00t_n1_7.architecture")) { + std::fprintf(stderr, "vla(gr00tn1d7): %s is not a gr00t_n1_7 GGUF\n", ckpt_path.c_str()); + return nullptr; + } + if (!load_config(g, *m, m->cfg)) + return nullptr; std::printf("vla(gr00tn1d7): vit=Qwen3-VL %lldd×%lldL×%lldh (Conv3d patch %lld², temporal %lld; learned pos %lld + 2D rope; deepstack@{%lld,%lld,%lld}; merge÷%lld) " "lm=Qwen3-VL %lldd×%lldL (%lldq/%lldkv×%lld, θ=%g) vlsa=%lldL×%lldh×%lld dit=AlternateVLDiT %lldL×%lldh×%lld(inner %lld) attend_text_every_n=%lld " "in_emb=%lld horizon=%lld action_dim=%lld max_state=%lld N_steps=%lld embodiment=%lld resident=%s\n", @@ -259,13 +275,18 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, { const Backend b = backend_init("vla(gr00tn1d7)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; } ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(gr00tn1d7): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(gr00tn1d7): ggml_init(ctx_weights) failed\n"); + return nullptr; + } WeightLoader L("gr00tn1d7", g, m->ctx_weights, m->matmul_type); @@ -279,18 +300,23 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, m->aex.declare(L, "aex"); m->dit.declare(L, "aex.dit", true, m->dit_interleave != 0); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(gr00tn1d7): QKV-fused DiT (self Wqkv / cross Wkv)\n"); std::printf("vla(gr00tn1d7): weights resident in %.2f GiB (%s) - incl. Qwen3-VL vision tower + deepstack + vl_self_attention; embodiment id %lld\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->matmul_type), (long long) m->aex.embodiment_id); - if (!m->build_caches()) { std::fprintf(stderr, "vla(gr00tn1d7): build_caches failed\n"); return nullptr; } + if (!m->build_caches()) { + std::fprintf(stderr, "vla(gr00tn1d7): build_caches failed\n"); + return nullptr; + } return m; } bool Gr00tN1d7ModelArch::build_caches() { - if (caches_ready) return true; + if (caches_ready) + return true; const int64_t side = image_target_size, ps = patch_size, m2 = spatial_merge; const int64_t grid = side / ps; const int64_t hd_vit = vit_hidden / vit_heads; @@ -300,7 +326,10 @@ bool Gr00tN1d7ModelArch::build_caches() { merge_block_coords(grid, grid, m2, c_grow, c_gcol); vit_rope_tables(c_grow, c_gcol, hd_vit, (double) vit_rope_base, c_rope_cos, c_rope_sin); - if (!io.open(gguf_path)) { std::fprintf(stderr, "vla(gr00tn1d7): build_caches: io.open(%s) failed\n", gguf_path.c_str()); return false; } + if (!io.open(gguf_path)) { + std::fprintf(stderr, "vla(gr00tn1d7): build_caches: io.open(%s) failed\n", gguf_path.c_str()); + return false; + } std::vector pos_table = io.read_f32("vit.pos_embd"); if (pos_table.empty() || (int64_t) pos_table.size() != vit_num_pos * vit_hidden) { std::fprintf(stderr, "vla(gr00tn1d7): build_caches: vit.pos_embd unreadable\n"); return false; @@ -341,11 +370,13 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { if (in.precomputed_img_emb && in.n_img_views > 0) { n_views = in.n_img_views; img_emb_ptr = in.precomputed_img_emb; - for (int j = 0; j < 3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); + for (int j = 0; j < 3; ++j) + ds_host[j].assign((size_t) n_views * K * H, 0.0f); } else if (in.images && in.n_images > 0) { n_views = in.n_images; img_emb_host.assign((size_t) n_views * K * H, 0.0f); - for (int j = 0; j < 3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); + for (int j = 0; j < 3; ++j) + ds_host[j].assign((size_t) n_views * K * H, 0.0f); ggml_context * VC = vision_scratch.reset((size_t) 512 * 1024 * 1024); if (!VC) { std::fprintf(stderr, "vla(gr00tn1d7): ggml_init(vision ctx) failed\n"); return {}; } @@ -360,29 +391,43 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < vit_layers; ++i) { h = build_vit_layer(VC, vit.blk[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); ggml_set_output(h); - for (int j = 0; j < 3; ++j) if (i == deepstack_idx[j]) stash[j] = h; + for (int j = 0; j < 3; ++j) + if (i == deepstack_idx[j]) + stash[j] = h; } ggml_tensor * ds_out[3]; - for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } + for (int j = 0; j < 3; ++j) { + ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); + ggml_set_output(ds_out[j]); + } ggml_tensor * vit_embeds = build_merger(VC, vit.merger, h, vit_hidden, m2, connector_ln_eps, true); ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); - for (int j = 0; j < 3; ++j) ggml_build_forward_expand(vg, ds_out[j]); + for (int j = 0; j < 3; ++j) + ggml_build_forward_expand(vg, ds_out[j]); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(gr00tn1d7): vision gallocr alloc failed\n"); return {}; } const auto tv0 = std::chrono::steady_clock::now(); std::vector patches; bool vok = true; for (int64_t v = 0; v < n_views && vok; ++v) { - if (!preprocess_image_patches("gr00tn1d7", in.images[v], side, ps, temporal_patch, grow, gcol, patches)) { vok = false; break; } + if (!preprocess_image_patches("gr00tn1d7", in.images[v], side, ps, temporal_patch, grow, gcol, patches)) { + vok = false; + break; + } ggml_backend_tensor_set(t_pos, pos_interp.data(), 0, ggml_nbytes(t_pos)); ggml_backend_tensor_set(t_cos, rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, rope_sin.data(), 0, ggml_nbytes(t_sin)); ggml_backend_tensor_set(t_patches, patches.data(), 0, ggml_nbytes(t_patches)); - if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d7): vision compute failed\n"); vok = false; break; } + if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(gr00tn1d7): vision compute failed\n"); + vok = false; + break; + } ggml_backend_tensor_get(vit_embeds, img_emb_host.data() + v * K * H, 0, ggml_nbytes(vit_embeds)); - for (int j = 0; j < 3; ++j) ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); + for (int j = 0; j < 3; ++j) + ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); } stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); if (!vok) return {}; @@ -394,13 +439,17 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector input_ids; int64_t n_img_slots = 0; - for (int j = 0; j < in.n_lang; ++j) if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_img_slots; + for (int j = 0; j < in.n_lang; ++j) + if (in.lang_tokens[j] == (int32_t) image_token_index) + ++n_img_slots; if (n_img_slots == n_img) { input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); } else if (n_img_slots == 0) { input_ids.reserve(n_img + in.n_lang); - for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); + for (int64_t i = 0; i < n_img; ++i) + input_ids.push_back((int32_t) image_token_index); + for (int j = 0; j < in.n_lang; ++j) + input_ids.push_back(in.lang_tokens[j]); } else { std::fprintf(stderr, "vla(gr00tn1d7): lang_tokens has %lld image-token slots but n_img=%lld; expected 0 (v1 fallback) or %lld (chat-template path)\n", (long long) n_img_slots, (long long) n_img, (long long) n_img); @@ -421,8 +470,10 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector image_pos_idx, text_pos_idx; image_pos_idx.reserve((size_t) n_img); text_pos_idx.reserve((size_t) (SEQ - n_img)); for (int64_t p = 0; p < SEQ; ++p) { - if (input_ids[p] == (int32_t) image_token_index) image_pos_idx.push_back((int32_t) p); - else text_pos_idx.push_back((int32_t) p); + if (input_ids[p] == (int32_t) image_token_index) + image_pos_idx.push_back((int32_t) p); + else + text_pos_idx.push_back((int32_t) p); } const int64_t SEQ_TXT = (int64_t) text_pos_idx.size(); if ((int64_t) image_pos_idx.size() != n_img) { @@ -440,14 +491,21 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { } std::vector x_init((size_t) AH * AD); - if (in.noise) std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); - else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); for (auto & v : x_init) v = nd(rng); } + if (in.noise) + std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); + else { + std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); + std::normal_distribution nd(0.f, 1.f); + for (auto & v : x_init) + v = nd(rng); + } // On by default: 16% faster, bit-identical. Set VLA_GR00T_GRAPH_CACHE=0 to opt out. // Dumping adds graph outputs, so it always rebuilds. const char * gc = std::getenv("VLA_GR00T_GRAPH_CACHE"); const bool use_cache = (!gc || std::strcmp(gc, "0") != 0) && !do_dump; - if (!use_cache) mg.release(); + if (!use_cache) + mg.release(); ggml_tensor * eagle = nullptr, * vl_embs = nullptr; std::vector lm_h_dump, vlsa_dump; @@ -460,12 +518,16 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1);ggml_set_input(t_state); ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); ggml_tensor * t_ds[3] = {nullptr,nullptr,nullptr}; - if (inject_deepstack) for (int j = 0; j < 3; ++j) { t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_ds[j]); } + if (inject_deepstack) for (int j = 0; j < 3; ++j) { + t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); + ggml_set_input(t_ds[j]); + } ggml_tensor * t_img_idx = ggml_new_tensor_1d(C, GGML_TYPE_I32, n_img); ggml_set_input(t_img_idx); ggml_tensor * t_txt_idx = (SEQ_TXT > 0) ? ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ_TXT) : nullptr; - if (t_txt_idx) ggml_set_input(t_txt_idx); + if (t_txt_idx) + ggml_set_input(t_txt_idx); std::vector t_tau(num_steps), t_tproj(num_steps); for (int64_t s = 0; s < num_steps; ++s) { t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); @@ -475,18 +537,28 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * h = t_embeds; for (int64_t i = 0; i < lm_layers; ++i) { h = lm.block(C, lm.blk[i], h, t_pos, t_lmmask, SEQ); - if (inject_deepstack && i < 3) h = ggml_add(C, h, t_ds[i]); - if (do_dump) { ggml_set_output(h); lm_h_dump.push_back(h); } + if (inject_deepstack && i < 3) + h = ggml_add(C, h, t_ds[i]); + if (do_dump) { + ggml_set_output(h); + lm_h_dump.push_back(h); + } } eagle = h; ggml_set_name(eagle, "eagle"); ggml_set_output(eagle); vl_embs = ggml_add(C, ggml_mul(C, ggml_norm(C, eagle, vlln_eps), vlln_w), vlln_b); - if (do_dump) { ggml_set_output(vl_embs); vlsa_dump.push_back(vl_embs); } + if (do_dump) { + ggml_set_output(vl_embs); + vlsa_dump.push_back(vl_embs); + } for (int64_t i = 0; i < vlsa_layers; ++i) { vl_embs = vlsa.block(C, vlsa.blk[i], vl_embs, SEQ); - if (do_dump) { ggml_set_output(vl_embs); vlsa_dump.push_back(vl_embs); } + if (do_dump) { + ggml_set_output(vl_embs); + vlsa_dump.push_back(vl_embs); + } } ggml_set_name(vl_embs, "vl_embs"); ggml_set_output(vl_embs); @@ -500,7 +572,8 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector Kc(dit_layers, nullptr), Vc(dit_layers, nullptr); for (int64_t i = 0; i < dit_layers; ++i) { - if (dit_interleave && (i % 2 == 1)) continue; + if (dit_interleave && (i % 2 == 1)) + continue; ggml_tensor * enc = (i % every2 == 0) ? vl_txt : vl_img; dit.kv(C, dit.blk[i], enc, &Kc[i], &Vc[i]); } @@ -515,9 +588,11 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * hh = sa; for (int64_t i = 0; i < dit_layers; ++i) { ggml_tensor * enc; - if (dit_interleave && (i % 2 == 1)) enc = nullptr; + if (dit_interleave && (i % 2 == 1)) + enc = nullptr; else if (i % every2 == 0) enc = vl_txt; - else enc = vl_img; + else + enc = vl_img; hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); } ggml_tensor * po = ggml_add(C, ggml_mul_mat(C, dit.po1W, ggml_silu(C, temb)), dit.po1b); @@ -558,7 +633,10 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; - for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { img_start = i; break; } + for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { + img_start = i; + break; + } const int64_t text_end = (img_start < 0) ? SEQ : img_start; const int64_t text_len = text_end - st; for (int64_t i = 0; i < text_len; ++i) { @@ -567,9 +645,14 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { pp[1 * SEQ + (st + i)] = p; pp[2 * SEQ + (st + i)] = p; } - if (img_start < 0) { st_idx += text_len; st = SEQ; break; } + if (img_start < 0) { + st_idx += text_len; + st = SEQ; + break; + } int64_t img_end = img_start; - while (img_end < SEQ && input_ids[img_end] == (int32_t) image_token_index) ++img_end; + while (img_end < SEQ && input_ids[img_end] == (int32_t) image_token_index) + ++img_end; const int64_t n_img_tokens = img_end - img_start; if (n_img_tokens % (llm_grid_h * llm_grid_w) != 0) { std::fprintf(stderr, "vla(gr00tn1d7): image run length %lld not a multiple of %lld (post-merge grid)\n", @@ -590,8 +673,10 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { } } int64_t max_image_pos = this_t - 1; - if (llm_grid_h - 1 > max_image_pos) max_image_pos = llm_grid_h - 1; - if (llm_grid_w - 1 > max_image_pos) max_image_pos = llm_grid_w - 1; + if (llm_grid_h - 1 > max_image_pos) + max_image_pos = llm_grid_h - 1; + if (llm_grid_w - 1 > max_image_pos) + max_image_pos = llm_grid_w - 1; st_idx = image_offset + max_image_pos + 1; st = img_end; } @@ -599,13 +684,22 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::memcpy(pp.data() + (size_t) 3 * SEQ, pp.data() + (size_t) 0 * SEQ, (size_t) SEQ * sizeof(int32_t)); ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } - if (c_mask_seq != SEQ) { build_causal_mask(SEQ, c_mask); c_mask_seq = SEQ; } + if (c_mask_seq != SEQ) { + build_causal_mask(SEQ, c_mask); + c_mask_seq = SEQ; + } ggml_backend_tensor_set(t_lmmask, c_mask.data(), 0, ggml_nbytes(t_lmmask)); - { std::vector st(max_state_dim, 0.0f); for (int64_t i = 0; i < max_state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); } + { + std::vector st(max_state_dim, 0.0f); + for (int64_t i = 0; i < max_state_dim; ++i) + st[i] = in.state ? in.state[i] : 0.0f; + ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); + } ggml_backend_tensor_set(t_x0, x_init.data(), 0, ggml_nbytes(t_x0)); if (inject_deepstack) for (int j = 0; j < 3; ++j) ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); ggml_backend_tensor_set(t_img_idx, image_pos_idx.data(), 0, ggml_nbytes(t_img_idx)); - if (t_txt_idx) ggml_backend_tensor_set(t_txt_idx, text_pos_idx.data(), 0, ggml_nbytes(t_txt_idx)); + if (t_txt_idx) + ggml_backend_tensor_set(t_txt_idx, text_pos_idx.data(), 0, ggml_nbytes(t_txt_idx)); for (int64_t s = 0; s < num_steps; ++s) { ggml_backend_tensor_set(t_tau[s], c_tau[(size_t) s].data(), 0, ggml_nbytes(t_tau[s])); ggml_backend_tensor_set(t_tproj[s], c_tproj[(size_t) s].data(), 0, ggml_nbytes(t_tproj[s])); @@ -627,20 +721,36 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_get(t, buf.data(), 0, buf.size() * sizeof(float)); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump, name, (long long) n0, (long long) n1); FILE * fp = std::fopen(path, "wb"); - if (fp) { std::fwrite(buf.data(), sizeof(float), buf.size(), fp); std::fclose(fp); std::fprintf(stderr, "vla(gr00tn1d7): dumped %s shape=(%lld,%lld) to %s\n", name, (long long) n1, (long long) n0, path); } + if (fp) { + std::fwrite(buf.data(), sizeof(float), buf.size(), fp); + std::fclose(fp); + std::fprintf(stderr, "vla(gr00tn1d7): dumped %s shape=(%lld,%lld) to %s\n", name, (long long) n1, (long long) n0, path); + } }; dump_t("eagle", eagle); dump_t("vl_embs", vl_embs); - for (size_t li = 0; li < lm_h_dump.size(); ++li) { char nm[32]; std::snprintf(nm, sizeof(nm), "lm_h_%02zu", li); dump_t(nm, lm_h_dump[li]); } + for (size_t li = 0; li < lm_h_dump.size(); ++li) { + char nm[32]; + std::snprintf(nm, sizeof(nm), "lm_h_%02zu", li); + dump_t(nm, lm_h_dump[li]); + } - for (size_t vi = 0; vi < vlsa_dump.size(); ++vi) { char nm[32]; std::snprintf(nm, sizeof(nm), "vlsa_%02zu", vi); dump_t(nm, vlsa_dump[vi]); } + for (size_t vi = 0; vi < vlsa_dump.size(); ++vi) { + char nm[32]; + std::snprintf(nm, sizeof(nm), "vlsa_%02zu", vi); + dump_t(nm, vlsa_dump[vi]); + } if (inject_deepstack) { auto dump_host = [&](const char * name, const float * data, int64_t n0, int64_t n1) { char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump, name, (long long) n0, (long long) n1); FILE * fp = std::fopen(path, "wb"); - if (fp) { std::fwrite(data, sizeof(float), (size_t) n0 * n1, fp); std::fclose(fp); std::fprintf(stderr, "vla(gr00tn1d7): dumped %s shape=(%lld,%lld) to %s\n", name, (long long) n1, (long long) n0, path); } + if (fp) { + std::fwrite(data, sizeof(float), (size_t) n0 * n1, fp); + std::fclose(fp); + std::fprintf(stderr, "vla(gr00tn1d7): dumped %s shape=(%lld,%lld) to %s\n", name, (long long) n1, (long long) n0, path); + } }; for (int64_t v = 0; v < n_views; ++v) { @@ -655,7 +765,8 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { } } } - if (!use_cache) mg.release(); + if (!use_cache) + mg.release(); stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); return out; } diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index ec41336..11343af 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -49,32 +49,51 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, }; const char * env = std::getenv("VLA_OPENVLA_OFT_UNNORM_KEY"); size_t suite_pos; - if (env) { suite = env; suite_pos = find_key(0, suite); } + if (env) { + suite = env; + suite_pos = find_key(0, suite); + } else { size_t b = js.find('{'); size_t q = js.find('"', b); size_t qe = js.find('"', q + 1); suite = js.substr(q + 1, qe - q - 1); suite_pos = q; } - if (suite_pos == std::string::npos) { std::fprintf(stderr, "vla(openvla_oft): suite '%s' not in stats\n", suite.c_str()); return false; } + if (suite_pos == std::string::npos) { + std::fprintf(stderr, "vla(openvla_oft): suite '%s' not in stats\n", suite.c_str()); + return false; + } size_t act = find_key(suite_pos, "action"); - if (act == std::string::npos) return false; + if (act == std::string::npos) + return false; auto read_arr = [&](const std::string & key, std::vector & out) -> bool { size_t k = find_key(act, key); if (k == std::string::npos) return false; size_t lb = js.find('[', k); size_t rb = js.find(']', lb); - if (lb == std::string::npos || rb == std::string::npos) return false; + if (lb == std::string::npos || rb == std::string::npos) + return false; out.clear(); size_t p = lb + 1; while (p < rb) { - while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) ++p; - if (p >= rb) break; + while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) + ++p; + if (p >= rb) + break; bool t = (js.compare(p, 4, "true") == 0), f = (js.compare(p, 5, "false") == 0); - if (t || f) { out.push_back(t ? 1.0f : 0.0f); p += t ? 4 : 5; } - else { out.push_back(std::strtof(js.c_str() + p, nullptr)); while (p < rb && js[p] != ',') ++p; } + if (t || f) { + out.push_back(t ? 1.0f : 0.0f); + p += t ? 4 : 5; + } + else { + out.push_back(std::strtof(js.c_str() + p, nullptr)); + while (p < rb && js[p] != ',') + ++p; + } } return true; }; std::vector mk; - if (!read_arr("q01", q01) || !read_arr("q99", q99)) return false; - if (!read_arr("mask", mk)) mk.assign(want, 1.0f); + if (!read_arr("q01", q01) || !read_arr("q99", q99)) + return false; + if (!read_arr("mask", mk)) + mk.assign(want, 1.0f); mask.assign(mk.size(), 1); for (size_t i = 0; i < mk.size(); ++i) mask[i] = mk[i] != 0.0f ? 1 : 0; return (int64_t) q01.size() == want && (int64_t) q99.size() == want; } @@ -87,9 +106,12 @@ struct HeadBlkW { ggml_tensor *lnw,*lnb,*linw,*linb; }; struct OpenVlaOftModelArch : public ModelArchBase { OpenVlaOftModelArch() : ModelArchBase(Arch::OPENVLA_OFT) {} ~OpenVlaOftModelArch() override { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } ggml_backend_t backend = nullptr; @@ -99,7 +121,9 @@ struct OpenVlaOftModelArch : public ModelArchBase { struct MainKey { int64_t seq=-1, n_views=-1, n_lang=-1; - bool operator==(const MainKey & o) const { return seq==o.seq && n_views==o.n_views && n_lang==o.n_lang; } + bool operator==(const MainKey & o) const { + return seq==o.seq && n_views==o.n_views && n_lang==o.n_lang; + } }; struct MainIO { ggml_tensor *t_ids=nullptr,*t_state=nullptr,*t_proj=nullptr,*act0=nullptr,*t_pos=nullptr,*norm_actions=nullptr; @@ -138,8 +162,12 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("openvla_oft"); - if (!g.open(ckpt_path)) return nullptr; - if (!g.has("openvla_oft.architecture")) { std::fprintf(stderr, "vla(openvla_oft): not an openvla_oft GGUF\n"); return nullptr; } + if (!g.open(ckpt_path)) + return nullptr; + if (!g.has("openvla_oft.architecture")) { + std::fprintf(stderr, "vla(openvla_oft): not an openvla_oft GGUF\n"); + return nullptr; + } auto U=[&](const char*k,int64_t&d){ if(g.has(k)) d=(int64_t)g.u32(k); }; auto F=[&](const char*k,float&d){ if(g.has(k)) d=g.f32(k); }; @@ -160,17 +188,23 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path // No empty_id: the reference zeroes the action-slot embeddings instead // (modeling_prismatic.py:891), which is what act0 below does. U("openvla_oft.tokens.stop_id",m->stop_id); - if (m->lm_head_dim==0) m->lm_head_dim = m->lm_hidden / m->n_q; + if (m->lm_head_dim==0) + m->lm_head_dim = m->lm_hidden / m->n_q; if (g.has("openvla_oft.statistics_json")) { if (!parse_stats(g.str("openvla_oft.statistics_json"), m->action_dim, m->q01, m->q99, m->unnorm_mask, m->suite)) - { std::fprintf(stderr, "vla(openvla_oft): failed to parse statistics_json\n"); return nullptr; } + { + std::fprintf(stderr, "vla(openvla_oft): failed to parse statistics_json\n"); + return nullptr; + } std::printf("vla(openvla_oft): unnorm suite = %s (q99 dim %zu)\n", m->suite.c_str(), m->q99.size()); } { const Backend b = backend_init("vla(openvla_oft)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; } @@ -204,9 +238,13 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path for(int i=0;ihead_blocks;++i){ auto&w=m->hblk[i]; char b[64]; auto N=[&](const char*s){ std::snprintf(b,sizeof(b),"aex.head.blk.%d.%s",i,s); return (const char*)b; }; w.lnw=f32(N("ln.weight")); w.lnb=f32(N("ln.bias")); w.linw=mm(N("lin.weight")); w.linb=f32(N("lin.bias")); } - if(!ok){ std::fprintf(stderr,"vla(openvla_oft): weight setup failed\n"); return nullptr; } + if(!ok){ + std::fprintf(stderr,"vla(openvla_oft): weight setup failed\n"); + return nullptr; + } - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(openvla_oft): weights resident %.2f GiB (%s) - DINOv2+SigLIP towers + Llama-2-7B + MLPResNet L1 head\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->mt)); @@ -291,7 +329,8 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { [&](ggml_context*C, MainIO & gio)->ggml_cgraph*{ ggml_tensor*t_ids=ggml_new_tensor_1d(C,GGML_TYPE_I32,L+1); ggml_set_input(t_ids); ggml_tensor*emb=ggml_get_rows(C,token_embd,t_ids); - if(emb->type!=GGML_TYPE_F32) emb=ggml_cast(C,emb,GGML_TYPE_F32); + if(emb->type!=GGML_TYPE_F32) + emb=ggml_cast(C,emb,GGML_TYPE_F32); ggml_tensor*bos =ggml_cont(C,ggml_view_2d(C,emb,HC,1,emb->nb[1],0)); ggml_tensor*rest=ggml_cont(C,ggml_view_2d(C,emb,HC,L-1,emb->nb[1],emb->nb[1])); ggml_tensor*stop=ggml_cont(C,ggml_view_2d(C,emb,HC,1,emb->nb[1],L*emb->nb[1])); @@ -360,14 +399,23 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { ggml_tensor*act0=gio.act0,*t_pos=gio.t_pos,*norm_actions=gio.norm_actions; { std::vector ids(L+1); - for(int64_t i=0;i pp(SEQ); for(int64_t i=0;i pp(SEQ); + for(int64_t i=0;i sv(proprio_dim,0.0f); for(int64_t i=0;i z((size_t)HC*n_act,0.0f); ggml_backend_tensor_set(act0,z.data(),0,ggml_nbytes(act0)); } + { + std::vector z((size_t)HC*n_act,0.0f); + ggml_backend_tensor_set(act0,z.data(),0,ggml_nbytes(act0)); + } if(ggml_backend_graph_compute(backend,gf)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(openvla_oft): main compute failed\n"); return {}; } std::vector na((size_t)action_dim*chunk); diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 3ba095a..bba826c 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -77,7 +77,9 @@ struct Pi0ModelArch : public ModelArchBase { struct MainKey { int64_t n_img=-1, n_lang=-1, nsteps=-1; - bool operator==(const MainKey & o) const { return n_img==o.n_img && n_lang==o.n_lang && nsteps==o.nsteps; } + bool operator==(const MainKey & o) const { + return n_img==o.n_img && n_lang==o.n_lang && nsteps==o.nsteps; + } }; struct MainIO { ggml_tensor *t_image_emb=nullptr,*t_lang_emb=nullptr,*t_prefix_pos=nullptr,*t_state=nullptr; @@ -193,8 +195,10 @@ ggml_tensor * build_gemma_layer( ggml_tensor * q_rope = rope_call(q_h); ggml_tensor * k_rope = rope_call(k_h); - if (k_out) *k_out = k_rope; - if (v_out) *v_out = v_h; + if (k_out) + *k_out = k_rope; + if (v_out) + *v_out = v_h; ggml_tensor * K_full = k_rope; ggml_tensor * V_full = v_h; @@ -252,7 +256,10 @@ ggml_tensor * build_embed_suffix(ggml_context * ctx, const Pi0ModelArch & m, bool load_config(const gguf_reader & g, Config & cfg) { auto need = [&](const char * k) { - if (!g.has(k)) { std::fprintf(stderr, "vla(pi0): gguf missing key %s\n", k); return false; } + if (!g.has(k)) { + std::fprintf(stderr, "vla(pi0): gguf missing key %s\n", k); + return false; + } return true; }; for (const char * k : {"pi0.hidden", "pi0.intermediate", "pi0.n_q_heads", "pi0.n_kv_heads", @@ -260,7 +267,8 @@ bool load_config(const gguf_reader & g, Config & cfg) { "pi0.chunk_size", "pi0.num_steps", "pi0.max_state_dim", "pi0.max_action_dim", "pi0.real_state_dim", "pi0.real_action_dim", "pi0.tokenizer_max_length", "pi0.min_period", "pi0.max_period"}) { - if (!need(k)) return false; + if (!need(k)) + return false; } cfg = Config{}; cfg.hidden = g.u32("pi0.hidden"); @@ -304,8 +312,14 @@ bool load_stats(gguf_reader & g, Pi0ModelArch & m) { m.action_std .assign(cfg.real_action_dim, 1.f); auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); - if (!t) { std::printf("vla(pi0): %s missing - identity\n", name); return; } - if (t->ne[0] != (int64_t) dst.size()) { std::printf("vla(pi0): %s dim mismatch - identity\n", name); return; } + if (!t) { + std::printf("vla(pi0): %s missing - identity\n", name); + return; + } + if (t->ne[0] != (int64_t) dst.size()) { + std::printf("vla(pi0): %s dim mismatch - identity\n", name); + return; + } const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size() * sizeof(float))) { // A short read leaves dst half-overwritten. @@ -323,9 +337,12 @@ bool load_stats(gguf_reader & g, Pi0ModelArch & m) { } Pi0ModelArch::~Pi0ModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr pi0_create(const std::string& mmproj_path, @@ -346,14 +363,16 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->ckpt_path_ = ckpt_path; m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); - if (!m->io.open(ckpt_path)) return nullptr; + if (!m->io.open(ckpt_path)) + return nullptr; gguf_reader & g = m->io; if (!g.has("pi0.architecture") || g.str("pi0.architecture") != "pi0") { std::fprintf(stderr, "vla(pi0): '%s' is not a π₀ GGUF (pi0.architecture missing/wrong)\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, m->cfg)) return nullptr; + if (!load_config(g, m->cfg)) + return nullptr; const Config & cfg = m->cfg; std::printf("vla(pi0): hidden=%lld inter=%lld heads=%lldq/%lldkv x%lld n_layers=%lld " "expert_h=%lld expert_inter=%lld chunk=%lld steps=%d real_state=%lld real_action=%lld " @@ -367,7 +386,9 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->n_threads = default_cpu_threads(); { const Backend b = backend_init("vla(pi0)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; // BF16 activations need BF16-resident weights and the CUDA BF16 GEMM path. @@ -389,7 +410,8 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, vu("pi0.vit_hidden", m->vit_hidden); vu("pi0.vit_layers", m->vit_layers); vu("pi0.vit_heads", m->vit_heads); vu("pi0.image_size", m->vit_image_size); vu("pi0.patch_size", m->vit_patch_size); vu("pi0.n_img_tokens", m->vit_n_tokens); - if (g.has("pi0.vit_ln_eps")) m->vit_ln_eps = g.f32("pi0.vit_ln_eps"); + if (g.has("pi0.vit_ln_eps")) + m->vit_ln_eps = g.f32("pi0.vit_ln_eps"); const int64_t grid = m->vit_image_size / m->vit_patch_size; if (grid * grid != m->vit_n_tokens || m->vit_n_tokens != cfg.n_img) { std::fprintf(stderr, "vla(pi0): vit geometry mismatch (grid^2=%lld n_img_tokens=%lld cfg.n_img=%lld)\n", @@ -401,7 +423,10 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, { ggml_init_params wp = { (size_t) 16 * 1024 * 1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(pi0): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(pi0): ggml_init(ctx_weights) failed\n"); + return nullptr; + } } WeightLoader L("pi0", g, m->ctx_weights, m->matmul_type); @@ -418,12 +443,14 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, m->W_at2 = L.f32("action_time_mlp_out.weight"); m->b_at2 = L.f32("action_time_mlp_out.bias"); m->W_aout = L.f32("action_out_proj.weight"); m->b_aout = L.f32("action_out_proj.bias"); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(pi0): resident weights = %.2f GiB\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0)); - if (!load_stats(g, *m)) return nullptr; + if (!load_stats(g, *m)) + return nullptr; std::printf("vla(pi0): model loaded (n_threads=%d)\n", m->n_threads); return m; } @@ -472,7 +499,8 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = mm_act(VC, mm_proj_w, h, act_type); - if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); + if (mm_proj_b) + proj = ggml_add(VC, proj, mm_proj_b); // read back to the host as F32 ggml_tensor * vit_emb = as_type(VC, ggml_scale(VC, proj, 1.0f / std::sqrt((float) proj->ne[0])), GGML_TYPE_F32); ggml_set_output(vit_emb); @@ -597,15 +625,21 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { { std::vector sh(max_sd, 0.f); - for (int64_t i = 0; i < max_sd; ++i) sh[i] = in.state ? in.state[i] : 0.f; + for (int64_t i = 0; i < max_sd; ++i) + sh[i] = in.state ? in.state[i] : 0.f; for (int64_t i = 0; i < cfg.real_state_dim && i < max_sd; ++i) sh[i] = (sh[i] - state_mean[i]) / (state_std[i] + cfg.norm_eps); ggml_backend_tensor_set(t_state, sh.data(), 0, ggml_nbytes(t_state)); } { std::vector x0h((size_t) max_ad * chunk); - if (in.noise) std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); - else { std::normal_distribution nd(0.f, 1.f); for (auto & v : x0h) v = nd(rng); } + if (in.noise) + std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); + else { + std::normal_distribution nd(0.f, 1.f); + for (auto & v : x0h) + v = nd(rng); + } ggml_backend_tensor_set(t_x0, x0h.data(), 0, ggml_nbytes(t_x0)); } { @@ -614,8 +648,12 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < n_suf; ++i) for (int64_t j = 0; j < n_total; ++j) { bool allowed; - if (j < n_prefix) allowed = true; - else { const int64_t s = j - n_prefix; allowed = (i == 0) ? (s == 0) : true; } + if (j < n_prefix) + allowed = true; + else { + const int64_t s = j - n_prefix; + allowed = (i == 0) ? (s == 0) : true; + } mk[i * n_total + j] = allowed ? 0.f : -INFINITY; } ggml_backend_tensor_set(t_full_mask, mk.data(), 0, ggml_nbytes(t_full_mask)); @@ -624,7 +662,8 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { const float timestep = 1.0f + (float) s * dt; const std::vector tv = sinusoidal_time_emb(timestep, hidden_ex, cfg.min_period, cfg.max_period); std::vector tile((size_t) hidden_ex * chunk); - for (int64_t c = 0; c < chunk; ++c) std::memcpy(tile.data() + c * hidden_ex, tv.data(), hidden_ex * sizeof(float)); + for (int64_t c = 0; c < chunk; ++c) + std::memcpy(tile.data() + c * hidden_ex, tv.data(), hidden_ex * sizeof(float)); ggml_backend_tensor_set(t_time[s], tile.data(), 0, ggml_nbytes(t_time[s])); } diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 3b9dda5..46fd095 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -90,7 +90,9 @@ struct Pi05ModelArch : public ModelArchBase { struct MainKey { int64_t n_img=-1, n_lang=-1, nsteps=-1; - bool operator==(const MainKey & o) const { return n_img==o.n_img && n_lang==o.n_lang && nsteps==o.nsteps; } + bool operator==(const MainKey & o) const { + return n_img==o.n_img && n_lang==o.n_lang && nsteps==o.nsteps; + } }; struct MainIO { ggml_tensor *t_image_emb=nullptr,*t_lang_emb=nullptr,*t_prefix_pos=nullptr; @@ -184,8 +186,10 @@ ggml_tensor * build_vlm_layer( ggml_tensor * q_rope = rope_call(q_h); ggml_tensor * k_rope = rope_call(k_h); - if (k_out) *k_out = k_rope; - if (v_out) *v_out = v_h; + if (k_out) + *k_out = k_rope; + if (v_out) + *v_out = v_h; ggml_tensor * Q = ggml_cont(ctx, ggml_permute(ctx, q_rope, 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(ctx, ggml_permute(ctx, k_rope, 0, 2, 1, 3)); @@ -222,7 +226,8 @@ ggml_tensor * build_adarms( ggml_tensor * out = ggml_add(ctx, ggml_add(ctx, normed, ggml_mul(ctx, normed, scale)), shift); - if (gate_out) *gate_out = gate; + if (gate_out) + *gate_out = gate; return out; } @@ -287,7 +292,10 @@ ggml_tensor * build_expert_layer( bool load_config(const gguf_reader & g, Config & cfg) { auto need = [&](const char * k) { - if (!g.has(k)) { std::fprintf(stderr, "vla(pi05): gguf missing key %s\n", k); return false; } + if (!g.has(k)) { + std::fprintf(stderr, "vla(pi05): gguf missing key %s\n", k); + return false; + } return true; }; for (const char * k : {"pi05.hidden", "pi05.intermediate", "pi05.n_q_heads", "pi05.n_kv_heads", @@ -295,7 +303,8 @@ bool load_config(const gguf_reader & g, Config & cfg) { "pi05.chunk_size", "pi05.num_steps", "pi05.max_state_dim", "pi05.max_action_dim", "pi05.real_state_dim", "pi05.real_action_dim", "pi05.tokenizer_max_length", "pi05.min_period", "pi05.max_period"}) { - if (!need(k)) return false; + if (!need(k)) + return false; } cfg = Config{}; cfg.hidden = g.u32("pi05.hidden"); @@ -339,8 +348,14 @@ bool load_stats(gguf_reader & g, Pi05ModelArch & m) { m.action_std .assign(cfg.real_action_dim, 1.f); auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); - if (!t) { std::printf("vla(pi05): %s missing - identity\n", name); return; } - if (t->ne[0] != (int64_t) dst.size()) { std::printf("vla(pi05): %s dim mismatch - identity\n", name); return; } + if (!t) { + std::printf("vla(pi05): %s missing - identity\n", name); + return; + } + if (t->ne[0] != (int64_t) dst.size()) { + std::printf("vla(pi05): %s dim mismatch - identity\n", name); + return; + } const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size() * sizeof(float))) { // A short read leaves dst half-overwritten. @@ -365,9 +380,12 @@ bool load_stats(gguf_reader & g, Pi05ModelArch & m) { } Pi05ModelArch::~Pi05ModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr pi05_create(const std::string& mmproj_path, @@ -387,14 +405,16 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, m->ckpt_path_ = ckpt_path; m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); - if (!m->io.open(ckpt_path)) return nullptr; + if (!m->io.open(ckpt_path)) + return nullptr; gguf_reader & g = m->io; if (!g.has("pi05.architecture") || g.str("pi05.architecture") != "pi05") { std::fprintf(stderr, "vla(pi05): '%s' is not a π0.5 GGUF (pi05.architecture missing/wrong)\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, m->cfg)) return nullptr; + if (!load_config(g, m->cfg)) + return nullptr; const Config & cfg = m->cfg; m->adarms_cond_dim = g.has("pi05.adarms_cond_dim") ? g.u32("pi05.adarms_cond_dim") : cfg.expert_h; m->quantile_norm = g.has("pi05.norm_mode") && g.str("pi05.norm_mode") == "quantiles"; @@ -411,7 +431,9 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, m->n_threads = default_cpu_threads(); { const Backend b = backend_init("vla(pi05)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; } @@ -422,7 +444,8 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, vu("pi05.vit_hidden", m->vit_hidden); vu("pi05.vit_layers", m->vit_layers); vu("pi05.vit_heads", m->vit_heads); vu("pi05.image_size", m->vit_image_size); vu("pi05.patch_size", m->vit_patch_size); vu("pi05.n_img_tokens", m->vit_n_tokens); - if (g.has("pi05.vit_ln_eps")) m->vit_ln_eps = g.f32("pi05.vit_ln_eps"); + if (g.has("pi05.vit_ln_eps")) + m->vit_ln_eps = g.f32("pi05.vit_ln_eps"); const int64_t grid = m->vit_image_size / m->vit_patch_size; if (grid * grid != m->vit_n_tokens || m->vit_n_tokens != cfg.n_img) { std::fprintf(stderr, "vla(pi05): vit geometry mismatch (grid^2=%lld n_img_tokens=%lld cfg.n_img=%lld)\n", @@ -434,7 +457,10 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, { ggml_init_params wp = { (size_t) 16 * 1024 * 1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(pi05): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(pi05): ggml_init(ctx_weights) failed\n"); + return nullptr; + } } ggml_context * W = m->ctx_weights; std::vector weights; @@ -443,7 +469,11 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, auto mk = [&](const char * name, ggml_type type, int n_dims, const int64_t * ne) -> ggml_tensor * { const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); missing = true; return nullptr; } + if (!gt) { + std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); + missing = true; + return nullptr; + } ggml_tensor * t = ggml_new_tensor(W, g.resident_type(gt, type), n_dims, ne); ggml_set_name(t, name); weights.push_back(t); @@ -451,12 +481,20 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, }; auto mk_mm = [&](const char * name) -> ggml_tensor * { const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); missing = true; return nullptr; } + if (!gt) { + std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); + missing = true; + return nullptr; + } return mk(name, m->matmul_type, GGML_MAX_DIMS, gt->ne); }; auto mk_f32 = [&](const char * name) -> ggml_tensor * { const ggml_tensor * gt = g.meta(name); - if (!gt) { std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); missing = true; return nullptr; } + if (!gt) { + std::fprintf(stderr, "vla(pi05): missing tensor %s\n", name); + missing = true; + return nullptr; + } return mk(name, GGML_TYPE_F32, GGML_MAX_DIMS, gt->ne); }; @@ -491,12 +529,14 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, m->W_tout = L.f32("time_mlp_out.weight"); m->b_tout = L.f32("time_mlp_out.bias"); m->W_aout = L.f32("action_out_proj.weight"); m->b_aout = L.f32("action_out_proj.bias"); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(pi05): resident weights = %.2f GiB\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0)); - if (!load_stats(g, *m)) return nullptr; + if (!load_stats(g, *m)) + return nullptr; std::printf("vla(pi05): model loaded (n_threads=%d)\n", m->n_threads); return m; } @@ -543,7 +583,8 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = ggml_mul_mat(VC, mm_proj_w, h); - if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); + if (mm_proj_b) + proj = ggml_add(VC, proj, mm_proj_b); ggml_tensor * vit_emb = ggml_scale(VC, proj, 1.0f / std::sqrt((float) proj->ne[0])); ggml_set_output(vit_emb); @@ -571,7 +612,8 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { // projector features. Inside this branch on purpose: precomputed_img_emb // replaces the tower and is already LM-ready. const float img_scale = (float) std::sqrt((double) hidden_pl); - for (float & x : img_emb_host) x *= img_scale; + for (float & x : img_emb_host) + x *= img_scale; } if (in.n_lang < 1 || !in.lang_tokens) { @@ -661,8 +703,13 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { } { std::vector x0h((size_t) max_ad * chunk); - if (in.noise) std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); - else { std::normal_distribution nd(0.f, 1.f); for (auto & v : x0h) v = nd(rng); } + if (in.noise) + std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); + else { + std::normal_distribution nd(0.f, 1.f); + for (auto & v : x0h) + v = nd(rng); + } ggml_backend_tensor_set(t_x0, x0h.data(), 0, ggml_nbytes(t_x0)); } for (int s = 0; s < num_steps; ++s) { diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 67aa957..857d843 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -63,7 +63,8 @@ struct safetensors { bool open(const std::string & path) { file.open(path, std::ios::binary); - if (!file) return false; + if (!file) + return false; uint64_t header_size = 0; file.read(reinterpret_cast(&header_size), sizeof(header_size)); std::string header_str(header_size, '\0'); @@ -71,7 +72,8 @@ struct safetensors { data_blob_start = sizeof(uint64_t) + header_size; json j = json::parse(header_str); for (auto it = j.begin(); it != j.end(); ++it) { - if (it.key() == "__metadata__") continue; + if (it.key() == "__metadata__") + continue; const auto & v = it.value(); st_tensor_info info; info.dtype = v.at("dtype").get(); @@ -105,7 +107,10 @@ struct safetensors { const size_t elsz = (info.dtype == "BF16") ? sizeof(ggml_bf16_t) : sizeof(float); size_t want = elsz; for (const int64_t d : info.shape) { - if (d < 0) { std::fprintf(stderr, "vla: negative dim for %s\n", name.c_str()); return false; } + if (d < 0) { + std::fprintf(stderr, "vla: negative dim for %s\n", name.c_str()); + return false; + } want *= (size_t) d; } if (info.off_end < info.off_begin || info.off_end - info.off_begin != want) { @@ -170,20 +175,26 @@ struct gguf_source { } ~gguf_source() { - if (fp) std::fclose(fp); - if (gctx) gguf_free(gctx); - if (meta_ctx) ggml_free(meta_ctx); + if (fp) + std::fclose(fp); + if (gctx) + gguf_free(gctx); + if (meta_ctx) + ggml_free(meta_ctx); } static bool shape_matches(const ggml_tensor * t, const std::vector & pt_shape) { const int nd_used = std::max(1, (int) pt_shape.size()); - if (nd_used > GGML_MAX_DIMS) return false; + if (nd_used > GGML_MAX_DIMS) + return false; for (int d = 0; d < (int) pt_shape.size(); ++d) { const int64_t expected = pt_shape[pt_shape.size() - 1 - d]; - if (t->ne[d] != expected) return false; + if (t->ne[d] != expected) + return false; } for (int d = (int) pt_shape.size(); d < GGML_MAX_DIMS; ++d) { - if (t->ne[d] != 1) return false; + if (t->ne[d] != 1) + return false; } return true; } @@ -207,10 +218,12 @@ struct gguf_source { return false; } if (t->type == GGML_TYPE_F32) { - if (std::fread(dst, 1, bytes, fp) != bytes) return false; + if (std::fread(dst, 1, bytes, fp) != bytes) + return false; } else if (t->type == GGML_TYPE_BF16) { std::vector tmp(bytes / sizeof(ggml_bf16_t)); - if (std::fread(tmp.data(), 1, bytes, fp) != bytes) return false; + if (std::fread(tmp.data(), 1, bytes, fp) != bytes) + return false; ggml_bf16_to_fp32_row(tmp.data(), dst, tmp.size()); } else { std::fprintf(stderr, "vla: gguf unsupported dtype %d for %s\n", @@ -236,7 +249,8 @@ struct gguf_source { return false; } const size_t offset = data_off + gguf_get_tensor_offset(gctx, id); - if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) return false; + if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) + return false; return std::fread(dst, 1, bytes, fp) == bytes; } @@ -246,11 +260,21 @@ struct gguf_source { bool has_key(const char * key) const { return find_key(key) >= 0; } - uint32_t get_u32(const char * key) const { return gguf_get_val_u32(gctx, find_key(key)); } - int32_t get_i32(const char * key) const { return gguf_get_val_i32(gctx, find_key(key)); } - float get_f32(const char * key) const { return gguf_get_val_f32(gctx, find_key(key)); } - double get_f64(const char * key) const { return gguf_get_val_f64(gctx, find_key(key)); } - std::string get_str(const char * key) const { return gguf_get_val_str(gctx, find_key(key)); } + uint32_t get_u32(const char * key) const { + return gguf_get_val_u32(gctx, find_key(key)); + } + int32_t get_i32(const char * key) const { + return gguf_get_val_i32(gctx, find_key(key)); + } + float get_f32(const char * key) const { + return gguf_get_val_f32(gctx, find_key(key)); + } + double get_f64(const char * key) const { + return gguf_get_val_f64(gctx, find_key(key)); + } + std::string get_str(const char * key) const { + return gguf_get_val_str(gctx, find_key(key)); + } bool has_tensor(const char * name) const { return ggml_get_tensor(meta_ctx, name) != nullptr; @@ -573,7 +597,8 @@ bool load_config_from_gguf(const gguf_source & st, Config & cfg) { "smolvla.real_state_dim", "smolvla.real_action_dim", "smolvla.self_attn_every_n_layers", "smolvla.tokenizer_max_length", "smolvla.min_period", "smolvla.max_period"}) { - if (!need(k)) return false; + if (!need(k)) + return false; } cfg.hidden = st.get_u32("smolvla.hidden"); @@ -626,15 +651,24 @@ std::string hf_to_gguf(const std::string & n) { static const char * MODEL_PFX = "model."; auto map_suffix = [](const std::string & s) -> std::string { - if (s == "input_layernorm.weight") return "attn_norm.weight"; - if (s == "self_attn.q_proj.weight") return "attn_q.weight"; - if (s == "self_attn.k_proj.weight") return "attn_k.weight"; - if (s == "self_attn.v_proj.weight") return "attn_v.weight"; - if (s == "self_attn.o_proj.weight") return "attn_o.weight"; - if (s == "post_attention_layernorm.weight") return "ffn_norm.weight"; - if (s == "mlp.gate_proj.weight") return "ffn_gate.weight"; - if (s == "mlp.up_proj.weight") return "ffn_up.weight"; - if (s == "mlp.down_proj.weight") return "ffn_down.weight"; + if (s == "input_layernorm.weight") + return "attn_norm.weight"; + if (s == "self_attn.q_proj.weight") + return "attn_q.weight"; + if (s == "self_attn.k_proj.weight") + return "attn_k.weight"; + if (s == "self_attn.v_proj.weight") + return "attn_v.weight"; + if (s == "self_attn.o_proj.weight") + return "attn_o.weight"; + if (s == "post_attention_layernorm.weight") + return "ffn_norm.weight"; + if (s == "mlp.gate_proj.weight") + return "ffn_gate.weight"; + if (s == "mlp.up_proj.weight") + return "ffn_up.weight"; + if (s == "mlp.down_proj.weight") + return "ffn_down.weight"; return s; }; auto starts_with = [](const std::string & s, const char * pfx) -> bool { @@ -651,9 +685,11 @@ std::string hf_to_gguf(const std::string & n) { auto layer_translate = [&](const std::string & rest, const char * dst_blk) -> std::string { - if (!starts_with(rest, "layers.")) return n; + if (!starts_with(rest, "layers.")) + return n; const size_t end_i = rest.find('.', 7); - if (end_i == std::string::npos) return n; + if (end_i == std::string::npos) + return n; const std::string idx = rest.substr(7, end_i - 7); const std::string suf = rest.substr(end_i + 1); return std::string(dst_blk) + ".blk." + idx + "." + map_suffix(suf); @@ -671,18 +707,25 @@ std::string hf_to_gguf(const std::string & n) { return "mm.fc.weight"; if (starts_with(n, VIS_PFX)) { const std::string rest = n.substr(std::strlen(VIS_PFX)); - if (rest == "embeddings.patch_embedding.weight") return "vit.patch_embd.weight"; - if (rest == "embeddings.patch_embedding.bias") return "vit.patch_embd.bias"; - if (rest == "embeddings.position_embedding.weight") return "vit.pos_embd"; - if (rest == "post_layernorm.weight") return "vit.post_ln.weight"; - if (rest == "post_layernorm.bias") return "vit.post_ln.bias"; + if (rest == "embeddings.patch_embedding.weight") + return "vit.patch_embd.weight"; + if (rest == "embeddings.patch_embedding.bias") + return "vit.patch_embd.bias"; + if (rest == "embeddings.position_embedding.weight") + return "vit.pos_embd"; + if (rest == "post_layernorm.weight") + return "vit.post_ln.weight"; + if (rest == "post_layernorm.bias") + return "vit.post_ln.bias"; if (starts_with(rest, "encoder.layers.")) { const size_t e = rest.find('.', 15); - if (e == std::string::npos) return n; + if (e == std::string::npos) + return n; const std::string idx = rest.substr(15, e - 15); const std::string suf = rest.substr(e + 1); std::string ds; - if (suf == "layer_norm1.weight") ds = "ln1.weight"; + if (suf == "layer_norm1.weight") + ds = "ln1.weight"; else if (suf == "layer_norm1.bias") ds = "ln1.bias"; else if (suf == "layer_norm2.weight") ds = "ln2.weight"; else if (suf == "layer_norm2.bias") ds = "ln2.bias"; @@ -698,7 +741,8 @@ std::string hf_to_gguf(const std::string & n) { else if (suf == "mlp.fc1.bias") ds = "fc1.bias"; else if (suf == "mlp.fc2.weight") ds = "fc2.weight"; else if (suf == "mlp.fc2.bias") ds = "fc2.bias"; - else return n; + else + return n; return "vit.blk." + idx + "." + ds; } return n; @@ -721,10 +765,13 @@ ggml_tensor * rope_q_or_k(ggml_context * ctx, ggml_tensor * x, 32.f, 1.f); } -static inline bool tower_mm_f32_prec() { return vla::mm_prec_f32_enabled(); } +static inline bool tower_mm_f32_prec() { + return vla::mm_prec_f32_enabled(); +} static inline ggml_tensor * mm_w(ggml_context * ctx, ggml_tensor * w, ggml_tensor * x) { ggml_tensor * r = ggml_mul_mat(ctx, w, x); - if (tower_mm_f32_prec()) ggml_mul_mat_set_prec(r, GGML_PREC_F32); + if (tower_mm_f32_prec()) + ggml_mul_mat_set_prec(r, GGML_PREC_F32); return r; } @@ -846,7 +893,8 @@ namespace { static void vram_probe(ggml_backend_t backend, const char * label) { ggml_backend_dev_t dev = ggml_backend_get_device(backend); - if (!dev) return; + if (!dev) + return; size_t free_b = 0, total_b = 0; ggml_backend_dev_memory(dev, &free_b, &total_b); static size_t prev_free = 0; @@ -867,10 +915,14 @@ static void vram_probe(ggml_backend_t backend, const char * label) { static ggml_type resolve_weight_dtype() { const char * e = std::getenv("VLA_WEIGHT_DTYPE"); - if (!e) return GGML_TYPE_BF16; - if (std::strcmp(e, "f32") == 0) return GGML_TYPE_F32; - if (std::strcmp(e, "bf16") == 0) return GGML_TYPE_BF16; - if (std::strcmp(e, "f16") == 0) return GGML_TYPE_F16; + if (!e) + return GGML_TYPE_BF16; + if (std::strcmp(e, "f32") == 0) + return GGML_TYPE_F32; + if (std::strcmp(e, "bf16") == 0) + return GGML_TYPE_BF16; + if (std::strcmp(e, "f16") == 0) + return GGML_TYPE_F16; std::fprintf(stderr, "vla: unknown VLA_WEIGHT_DTYPE='%s', using bf16\n", e); return GGML_TYPE_BF16; } @@ -930,7 +982,10 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, { const Backend b = backend_init("vla", default_cpu_threads()); - if (!b.handle) { delete m; return nullptr; } + if (!b.handle) { + delete m; + return nullptr; + } m->backend = b.handle; } vram_probe(m->backend, "after backend init"); @@ -946,7 +1001,8 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, vu("smolvla.vit_heads", m->vit_heads); vu("smolvla.patch_size", m->vit_patch); vu("smolvla.image_size", m->vit_image); vu("smolvla.vit_pixel_shuffle", m->vit_scale); vu("smolvla.n_img_tokens", m->vit_n_tokens); vu("smolvla.vit_inter", m->vit_inter); - if (gst.has_key("smolvla.vit_ln_eps")) m->vit_ln_eps = gst.get_f32("smolvla.vit_ln_eps"); + if (gst.has_key("smolvla.vit_ln_eps")) + m->vit_ln_eps = gst.get_f32("smolvla.vit_ln_eps"); } { const int64_t grid = m->vit_image / m->vit_patch; @@ -974,7 +1030,8 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, for (const auto & kv : st.tensors) { if (kv.first.compare(0, prefix.size(), prefix) == 0) { const int idx = std::atoi(kv.first.c_str() + prefix.size()); - if (idx > max_layer) max_layer = idx; + if (idx > max_layer) + max_layer = idx; } } if (max_layer < 0) { @@ -1233,10 +1290,16 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, }; for (auto & p : pending_f32) { - if (!stream_f32(p.name, p.t, p.shape)) { delete m; return nullptr; } + if (!stream_f32(p.name, p.t, p.shape)) { + delete m; + return nullptr; + } } for (auto & p : pending_bf16) { - if (!stream_bf16(p.name, p.t)) { delete m; return nullptr; } + if (!stream_bf16(p.name, p.t)) { + delete m; + return nullptr; + } } { @@ -1411,11 +1474,16 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { SmolVLAModelArch::~SmolVLAModelArch() { - if (galloc) ggml_gallocr_free(galloc); - if (ctx_compute) ggml_free(ctx_compute); - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (galloc) + ggml_gallocr_free(galloc); + if (ctx_compute) + ggml_free(ctx_compute); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } namespace { @@ -1490,7 +1558,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector chw, post_host((size_t) H * n_patches), shuf_host((size_t) c4 * K); bool vok = true; for (int v = 0; v < n_views && vok; ++v) { - if (!preprocess_image_chw("smolvla", in.images[v], m->vit_image, chw)) { vok = false; break; } + if (!preprocess_image_chw("smolvla", in.images[v], m->vit_image, chw)) { + vok = false; + break; + } ggml_backend_tensor_set(t_px, chw.data(), 0, ggml_nbytes(t_px)); if (ggml_backend_graph_compute(m->backend, gA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): vision compute A failed (view %d)\n", v); vok = false; break; @@ -1532,8 +1603,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { if (in.timing_detail == TimingDetail::NONE) { if (m->gf_cached == nullptr || m->cached_n_views != n_views) { - if (m->galloc) ggml_gallocr_free(m->galloc); - if (m->ctx_compute) ggml_free(m->ctx_compute); + if (m->galloc) + ggml_gallocr_free(m->galloc); + if (m->ctx_compute) + ggml_free(m->ctx_compute); m->galloc = nullptr; m->ctx_compute = nullptr; m->gf_cached = nullptr; @@ -1551,7 +1624,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { const int64_t pad_end = cfg.n_img + n_lang_max; std::vector state_host(cfg.max_state_dim, 0.0f); - if (in.state) std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); + if (in.state) + std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { state_host[i] = (state_host[i] - m->state_mean[i]) / (m->state_std[i] + cfg.norm_eps); } @@ -1561,7 +1635,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::memcpy(noise_host.data(), in.noise, noise_host.size() * sizeof(float)); } else { std::normal_distribution dist(0.f, 1.f); - for (auto & v : noise_host) v = dist(m->rng); + for (auto & v : noise_host) + v = dist(m->rng); } std::vector lang_host(n_lang_max, 0); @@ -1574,8 +1649,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { for (int64_t i = 0; i < n_prefix_max; ++i) { for (int64_t j = 0; j < n_prefix_max; ++j) { bool blocked = false; - if ((i < n_prefix_max - 1) && (j == n_prefix_max - 1)) blocked = true; - if (j >= pad_start && j < pad_end) blocked = true; + if ((i < n_prefix_max - 1) && (j == n_prefix_max - 1)) + blocked = true; + if (j >= pad_start && j < pad_end) + blocked = true; mask_prefill_host[i * n_prefix_max + j] = blocked ? -INFINITY : 0.f; } pos_prefill_host[i] = (i == n_prefix_max - 1) @@ -1678,7 +1755,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_tensor * pos_rebased = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, cfg.n_suffix); std::vector state_host(cfg.max_state_dim, 0.0f); - if (in.state) std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); + if (in.state) + std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { state_host[i] = (state_host[i] - m->state_mean[i]) / (m->state_std[i] + cfg.norm_eps); } @@ -1688,7 +1766,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::memcpy(noise_host.data(), in.noise, noise_host.size() * sizeof(float)); } else { std::normal_distribution dist(0.f, 1.f); - for (auto & v : noise_host) v = dist(m->rng); + for (auto & v : noise_host) + v = dist(m->rng); } std::vector mask_prefill_host(cfg.n_prefix * cfg.n_prefix); @@ -1708,8 +1787,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { for (int64_t i = 0; i < cfg.n_suffix; ++i) { for (int64_t j = 0; j < cfg.n_full; ++j) { bool blocked; - if (j < cfg.n_prefix) blocked = false; - else blocked = ((j - cfg.n_prefix) > i); + if (j < cfg.n_prefix) + blocked = false; + else + blocked = ((j - cfg.n_prefix) > i); mask_full_host[i * cfg.n_full + j] = blocked ? -INFINITY : 0.f; } pos_full_host [i] = static_cast(cfg.n_prefix + i); @@ -1894,7 +1975,8 @@ std::unique_ptr smolvla_create(const std::string& mmproj_path, const std::string& config_path, const Options& opts) { SmolVLAModelArch* raw = smolvla_load_impl(mmproj_path, ckpt_path, config_path); - if (!raw) return nullptr; + if (!raw) + return nullptr; return std::unique_ptr(raw); } diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 1595788..81d982c 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -51,32 +51,51 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, const char * env = std::getenv("VLA_ADAPTER_UNNORM_KEY"); size_t suite_pos; - if (env) { suite = env; suite_pos = find_key(0, suite); } + if (env) { + suite = env; + suite_pos = find_key(0, suite); + } else { size_t b = js.find('{'); size_t q = js.find('"', b); size_t qe = js.find('"', q + 1); suite = js.substr(q + 1, qe - q - 1); suite_pos = q; } - if (suite_pos == std::string::npos) { std::fprintf(stderr, "vla(vla_adapter): suite '%s' not in stats\n", suite.c_str()); return false; } + if (suite_pos == std::string::npos) { + std::fprintf(stderr, "vla(vla_adapter): suite '%s' not in stats\n", suite.c_str()); + return false; + } size_t act = find_key(suite_pos, "action"); - if (act == std::string::npos) return false; + if (act == std::string::npos) + return false; auto read_arr = [&](const std::string & key, std::vector & out) -> bool { size_t k = find_key(act, key); if (k == std::string::npos) return false; size_t lb = js.find('[', k); size_t rb = js.find(']', lb); - if (lb == std::string::npos || rb == std::string::npos) return false; + if (lb == std::string::npos || rb == std::string::npos) + return false; out.clear(); size_t p = lb + 1; while (p < rb) { - while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) ++p; - if (p >= rb) break; + while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) + ++p; + if (p >= rb) + break; bool t = (js.compare(p, 4, "true") == 0), f = (js.compare(p, 5, "false") == 0); - if (t || f) { out.push_back(t ? 1.0f : 0.0f); p += t ? 4 : 5; } - else { out.push_back(std::strtof(js.c_str() + p, nullptr)); while (p < rb && js[p] != ',') ++p; } + if (t || f) { + out.push_back(t ? 1.0f : 0.0f); + p += t ? 4 : 5; + } + else { + out.push_back(std::strtof(js.c_str() + p, nullptr)); + while (p < rb && js[p] != ',') + ++p; + } } return true; }; std::vector mk; - if (!read_arr("q01", q01) || !read_arr("q99", q99)) return false; - if (!read_arr("mask", mk)) mk.assign(want, 1.0f); + if (!read_arr("q01", q01) || !read_arr("q99", q99)) + return false; + if (!read_arr("mask", mk)) + mk.assign(want, 1.0f); mask.assign(mk.size(), 1); for (size_t i = 0; i < mk.size(); ++i) mask[i] = mk[i] != 0.0f ? 1 : 0; return (int64_t) q01.size() == want && (int64_t) q99.size() == want; } @@ -89,9 +108,12 @@ struct HeadBlkW { ggml_tensor *Wq,*bq,*Wks,*bks,*Wvs,*bvs,*Wka,*bka,*Wva,*bva,* struct VlaAdapterModelArch : public ModelArchBase { VlaAdapterModelArch() : ModelArchBase(Arch::VLA_ADAPTER) {} ~VlaAdapterModelArch() override { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } ggml_backend_t backend = nullptr; @@ -101,7 +123,9 @@ struct VlaAdapterModelArch : public ModelArchBase { struct MainKey { int64_t seq=-1, n_views=-1, nprompt=-1; - bool operator==(const MainKey & o) const { return seq==o.seq && n_views==o.n_views && nprompt==o.nprompt; } + bool operator==(const MainKey & o) const { + return seq==o.seq && n_views==o.n_views && nprompt==o.nprompt; + } }; struct MainIO { ggml_tensor *t_ids=nullptr,*t_proj=nullptr,*t_pos=nullptr,*t_mask=nullptr; @@ -144,7 +168,9 @@ static ggml_tensor* hrot(ggml_context*C, ggml_tensor*x, int64_t HD){ ggml_tensor*od=ggml_cont(C,ggml_view_4d(C,xp,1,HD/2,L,H,xp->nb[1],xp->nb[2],xp->nb[3],xp->nb[0])); return ggml_reshape_3d(C,ggml_concat(C,ggml_scale(C,od,-1.0f),ev,0),HD,L,H); } -static ggml_tensor* hheads(ggml_context*C, ggml_tensor*p, int64_t HD, int64_t NH){ return ggml_cont(C,ggml_permute(C,ggml_reshape_3d(C,p,HD,NH,p->ne[1]),0,2,1,3)); } +static ggml_tensor* hheads(ggml_context*C, ggml_tensor*p, int64_t HD, int64_t NH){ + return ggml_cont(C,ggml_permute(C,ggml_reshape_3d(C,p,HD,NH,p->ne[1]),0,2,1,3)); +} static ggml_tensor* hrope(ggml_context*C, ggml_tensor*x, ggml_tensor*cs, ggml_tensor*sn, int64_t HD){ ggml_tensor*c=ggml_reshape_3d(C,cs,HD,x->ne[1],1),*s=ggml_reshape_3d(C,sn,HD,x->ne[1],1); return ggml_add(C,ggml_mul(C,x,c),ggml_mul(C,hrot(C,x,HD),s)); @@ -162,8 +188,12 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("vla_adapter"); - if (!g.open(ckpt_path)) return nullptr; - if (!g.has("vla_adapter.architecture")) { std::fprintf(stderr, "vla(vla_adapter): not a vla_adapter GGUF\n"); return nullptr; } + if (!g.open(ckpt_path)) + return nullptr; + if (!g.has("vla_adapter.architecture")) { + std::fprintf(stderr, "vla(vla_adapter): not a vla_adapter GGUF\n"); + return nullptr; + } auto U=[&](const char*k,int64_t&d){ if(g.has(k)) d=(int64_t)g.u32(k); }; auto F=[&](const char*k,float&d){ if(g.has(k)) d=g.f32(k); }; @@ -194,13 +224,18 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path if (g.has("vla_adapter.statistics_json")) { if (!parse_stats(g.str("vla_adapter.statistics_json"), m->action_dim, m->q01, m->q99, m->unnorm_mask, m->suite)) - { std::fprintf(stderr, "vla(vla_adapter): failed to parse statistics_json\n"); return nullptr; } + { + std::fprintf(stderr, "vla(vla_adapter): failed to parse statistics_json\n"); + return nullptr; + } std::printf("vla(vla_adapter): unnorm suite = %s (q99 dim %zu)\n", m->suite.c_str(), m->q99.size()); } { const Backend b = backend_init("vla(vla_adapter)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; } @@ -242,9 +277,13 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path w.flnw=f32(N("ffn_ln.weight")); w.flnb=f32(N("ffn_ln.bias")); w.flw=mm(N("ffn_lin.weight")); w.flb=f32(N("ffn_lin.bias")); std::vector gv=g.read_f32(N("gating")); w.rg = gv.empty()?0.0f:std::tanh(gv[0]); } (void)P; - if(!ok){ std::fprintf(stderr,"vla(vla_adapter): weight setup failed\n"); return nullptr; } + if(!ok){ + std::fprintf(stderr,"vla(vla_adapter): weight setup failed\n"); + return nullptr; + } - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(vla_adapter): weights resident %.2f GiB (%s) - DINOv2+SigLIP towers + Qwen2.5-0.5B + Bridge head\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->mt)); @@ -331,7 +370,8 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { [&](ggml_context*C, MainIO & gio)->ggml_cgraph*{ ggml_tensor*t_ids=ggml_new_tensor_1d(C,GGML_TYPE_I32,NPROMPT+num_tokens+1); ggml_set_input(t_ids); ggml_tensor*emb=ggml_get_rows(C,token_embd,t_ids); - if(emb->type!=GGML_TYPE_F32) emb=ggml_cast(C,emb,GGML_TYPE_F32); + if(emb->type!=GGML_TYPE_F32) + emb=ggml_cast(C,emb,GGML_TYPE_F32); ggml_tensor*aqf=action_queries->type==GGML_TYPE_F32?action_queries:ggml_cast(C,action_queries,GGML_TYPE_F32); ggml_tensor*pre=ggml_cont(C,ggml_view_2d(C,emb,HC,NPROMPT,emb->nb[1],0)); ggml_tensor*stop=ggml_cont(C,ggml_view_2d(C,emb,HC,1,emb->nb[1],(NPROMPT+num_tokens)*emb->nb[1])); @@ -364,7 +404,8 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { ggml_tensor*final_norm=ggml_mul(C,ggml_rms_norm(C,lout[lm_layers-1],lm_rms_eps),lm_out_norm); std::vector cond(head_blocks); - for(int i=0;i VlaAdapterModelArch::predict(const Inputs& in) { ggml_tensor*cT=gio.cT,*sT=gio.sT,*cA=gio.cA,*sA=gio.sA,*cK=gio.cK,*sK=gio.sK; { std::vector ids(NPROMPT+num_tokens+1); - for(int64_t i=0;i pp(SEQ); for(int64_t i=0;i pp(SEQ); + for(int64_t i=0;i mk; build_causal_mask(SEQ, mk); ggml_backend_tensor_set(t_mask,mk.data(),0,ggml_nbytes(t_mask)); } { std::vector sv(proprio_dim,0.0f); for(int64_t i=0;i zx((size_t)action_dim*HC*chunk,0.0f); ggml_backend_tensor_set(t_x0,zx.data(),0,ggml_nbytes(t_x0)); } + { + std::vector zx((size_t)action_dim*HC*chunk,0.0f); + ggml_backend_tensor_set(t_x0,zx.data(),0,ggml_nbytes(t_x0)); + } auto fill_cs=[&](ggml_tensor*cc,ggml_tensor*ss,int64_t Lh){ std::vector cb(HD*Lh),sb(HD*Lh); const int64_t half=HD/2; - for(int64_t t=0;t= 1) { m.num_steps = (int64_t) v; std::fprintf(stderr, "vla(vla_jepa): VLA_NUM_STEPS override → num_steps=%lld\n", (long long) v); } + if (end && *end == '\0' && v >= 1) { + m.num_steps = (int64_t) v; + std::fprintf(stderr, "vla(vla_jepa): VLA_NUM_STEPS override → num_steps=%lld\n", (long long) v); + } } F(fk("vit_ln_eps"), m.vit_ln_eps); F(fk("lm_rms_eps"), m.lm_rms_eps); F(fk("connector_ln_eps"), m.connector_ln_eps); F(fk("vit_rope_theta"), m.vit_rope_base); F(fk("dit_ln_eps"), m.dit_ln_eps); F(fk("dit_norm_out_eps"), m.dit_norm_out_eps); - if (g.has(fk("lm_rope_theta"))) m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); + if (g.has(fk("lm_rope_theta"))) + m.lm_rope_base = (float) g.f64(fk("lm_rope_theta")); // merge_block_coords only enumerates the patch grid exactly when the spatial // merge divides it; otherwise it emits rows past the position table. @@ -196,9 +204,12 @@ bool load_config(const gguf_reader & g, VlaJepaModelArch & m, Config & cfg) { } VlaJepaModelArch::~VlaJepaModelArch() { - if (weight_buf) ggml_backend_buffer_free(weight_buf); - if (ctx_weights) ggml_free(ctx_weights); - if (backend) ggml_backend_free(backend); + if (weight_buf) + ggml_backend_buffer_free(weight_buf); + if (ctx_weights) + ggml_free(ctx_weights); + if (backend) + ggml_backend_free(backend); } std::unique_ptr vla_jepa_create(const std::string& mmproj_path, @@ -213,9 +224,14 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); gguf_reader g("vla_jepa"); - if (!g.open(ckpt_path)) return nullptr; - if (!g.has("vla_jepa.architecture")) { std::fprintf(stderr, "vla(vla_jepa): %s is not a vla_jepa GGUF\n", ckpt_path.c_str()); return nullptr; } - if (!load_config(g, *m, m->cfg)) return nullptr; + if (!g.open(ckpt_path)) + return nullptr; + if (!g.has("vla_jepa.architecture")) { + std::fprintf(stderr, "vla(vla_jepa): %s is not a vla_jepa GGUF\n", ckpt_path.c_str()); + return nullptr; + } + if (!load_config(g, *m, m->cfg)) + return nullptr; std::printf("vla(vla_jepa): vit=Qwen3-VL %lldd×%lldL (deepstack@{%lld,%lld,%lld}, merge÷%lld) lm=Qwen3-VL %lldd×%lldL (%lldq/%lldkv×%lld, θ=%g) " "dit-B %lldL×%lldh×%lld(inner %lld, cross %lld, out %lld) horizon=%lld action_dim=%lld state_dim=%lld future=%lld N_steps=%lld resident=%s\n", (long long) m->vit_hidden, (long long) m->vit_layers, (long long) m->deepstack_idx[0], (long long) m->deepstack_idx[1], (long long) m->deepstack_idx[2], (long long) m->spatial_merge, @@ -226,13 +242,18 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, { const Backend b = backend_init("vla(vla_jepa)", m->n_threads); - if (!b.handle) { return nullptr; } + if (!b.handle) { + return nullptr; + } m->backend = b.handle; } ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); - if (!m->ctx_weights) { std::fprintf(stderr, "vla(vla_jepa): ggml_init(ctx_weights) failed\n"); return nullptr; } + if (!m->ctx_weights) { + std::fprintf(stderr, "vla(vla_jepa): ggml_init(ctx_weights) failed\n"); + return nullptr; + } WeightLoader L("vla_jepa", g, m->ctx_weights, m->matmul_type); @@ -251,16 +272,21 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, m->dit.declare(L, "ah.dit", false, false, "ah"); - if (!L.upload(m->backend, &m->weight_buf)) return nullptr; + if (!L.upload(m->backend, &m->weight_buf)) + return nullptr; std::printf("vla(vla_jepa): weights resident in %.2f GiB (%s)\n", ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->matmul_type)); - if (!m->build_caches()) { std::fprintf(stderr, "vla(vla_jepa): build_caches failed\n"); return nullptr; } + if (!m->build_caches()) { + std::fprintf(stderr, "vla(vla_jepa): build_caches failed\n"); + return nullptr; + } return m; } bool VlaJepaModelArch::build_caches() { - if (caches_ready) return true; + if (caches_ready) + return true; const int64_t side = image_target_size, ps = patch_size, m2 = spatial_merge; const int64_t grid = side / ps; const int64_t hd_vit = vit_hidden / vit_heads; @@ -269,7 +295,10 @@ bool VlaJepaModelArch::build_caches() { merge_block_coords(grid, grid, m2, c_grow, c_gcol); vit_rope_tables(c_grow, c_gcol, hd_vit, (double) vit_rope_base, c_rope_cos, c_rope_sin); - if (!io.open(gguf_path)) { std::fprintf(stderr, "vla(vla_jepa): build_caches: io.open(%s) failed\n", gguf_path.c_str()); return false; } + if (!io.open(gguf_path)) { + std::fprintf(stderr, "vla(vla_jepa): build_caches: io.open(%s) failed\n", gguf_path.c_str()); + return false; + } std::vector pos_table = io.read_f32("vit.pos_embd"); if (pos_table.empty() || (int64_t) pos_table.size() != vit_num_pos * vit_hidden) { std::fprintf(stderr, "vla(vla_jepa): build_caches: vit.pos_embd unreadable\n"); return false; @@ -299,17 +328,27 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { if (!caches_ready) { std::fprintf(stderr, "vla(vla_jepa): caches not ready\n"); return {}; } auto dump_t = [&](const char * name, ggml_tensor * t) { - if (!dump_prefix) return; + if (!dump_prefix) + return; const int64_t n0 = t->ne[0], n1 = t->ne[1]; std::vector buf((size_t) n0 * std::max(1, n1)); ggml_backend_tensor_get(t, buf.data(), 0, buf.size() * sizeof(float)); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, name, (long long) n0, (long long) n1); - FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(buf.data(), sizeof(float), buf.size(), fp); std::fclose(fp); } + FILE * fp = std::fopen(path, "wb"); if (fp) { + std::fwrite(buf.data(), sizeof(float), buf.size(), fp); + std::fclose(fp); + } }; std::vector x_init((size_t) AH * AD); - if (in.noise) std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); - else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); for (auto & v : x_init) v = nd(rng); } + if (in.noise) + std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); + else { + std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); + std::normal_distribution nd(0.f, 1.f); + for (auto & v : x_init) + v = nd(rng); + } std::vector cond_host((size_t) H * num_future, 0.0f); const char * cond_file = std::getenv("VLA_JEPA_COND"); @@ -329,7 +368,8 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { int64_t n_views = in.n_images; if (n_views <= 0) { std::fprintf(stderr, "vla(vla_jepa): no images in the request\n"); return {}; } std::vector img_emb_host((size_t) n_views * K * H), ds_host[3]; - for (int j = 0; j < 3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); + for (int j = 0; j < 3; ++j) + ds_host[j].assign((size_t) n_views * K * H, 0.0f); std::vector inj_patches; const char * patches_file = std::getenv("VLA_JEPA_PATCHES"); if (patches_file) { @@ -354,15 +394,21 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < vit_layers; ++i) { h = build_vit_layer(VC, vit.blk[i], h, t_cos, t_sin, n_patches, vit_heads, hd_vit, vit_hidden, vit_ln_eps); ggml_set_output(h); - for (int j = 0; j < 3; ++j) if (i == deepstack_idx[j]) stash[j] = h; + for (int j = 0; j < 3; ++j) + if (i == deepstack_idx[j]) + stash[j] = h; } ggml_tensor * ds_out[3]; - for (int j = 0; j < 3; ++j) { ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); ggml_set_output(ds_out[j]); } + for (int j = 0; j < 3; ++j) { + ds_out[j] = build_merger(VC, vit.deepstack[j], stash[j] ? stash[j] : h, vit_hidden, m2, connector_ln_eps, false); + ggml_set_output(ds_out[j]); + } ggml_tensor * vit_embeds = build_merger(VC, vit.merger, h, vit_hidden, m2, connector_ln_eps, true); ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); - for (int j = 0; j < 3; ++j) ggml_build_forward_expand(vg, ds_out[j]); + for (int j = 0; j < 3; ++j) + ggml_build_forward_expand(vg, ds_out[j]); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(vla_jepa): vision gallocr alloc failed\n"); return {}; } const auto tv0 = std::chrono::steady_clock::now(); std::vector patches; @@ -371,15 +417,23 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { if (!inj_patches.empty()) { ggml_backend_tensor_set(t_patches, inj_patches.data() + v * n_patches * vit_patch_flat, 0, ggml_nbytes(t_patches)); } else { - if (!preprocess_image_patches("vla_jepa", in.images[v], side, ps, temporal_patch, c_grow, c_gcol, patches)) { vok = false; break; } + if (!preprocess_image_patches("vla_jepa", in.images[v], side, ps, temporal_patch, c_grow, c_gcol, patches)) { + vok = false; + break; + } ggml_backend_tensor_set(t_patches, patches.data(), 0, ggml_nbytes(t_patches)); } ggml_backend_tensor_set(t_pos, c_pos_interp.data(), 0, ggml_nbytes(t_pos)); ggml_backend_tensor_set(t_cos, c_rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, c_rope_sin.data(), 0, ggml_nbytes(t_sin)); - if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): vision compute failed\n"); vok = false; break; } + if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "vla(vla_jepa): vision compute failed\n"); + vok = false; + break; + } ggml_backend_tensor_get(vit_embeds, img_emb_host.data() + v * K * H, 0, ggml_nbytes(vit_embeds)); - for (int j = 0; j < 3; ++j) ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); + for (int j = 0; j < 3; ++j) + ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data() + v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } } stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); @@ -388,13 +442,17 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector input_ids; int64_t n_img_slots = 0; - for (int j = 0; j < in.n_lang; ++j) if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_img_slots; + for (int j = 0; j < in.n_lang; ++j) + if (in.lang_tokens[j] == (int32_t) image_token_index) + ++n_img_slots; if (n_img_slots == n_img) { input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); } else if (n_img_slots == 0) { input_ids.reserve(n_img + in.n_lang); - for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); - for (int j = 0; j < in.n_lang; ++j) input_ids.push_back(in.lang_tokens[j]); + for (int64_t i = 0; i < n_img; ++i) + input_ids.push_back((int32_t) image_token_index); + for (int j = 0; j < in.n_lang; ++j) + input_ids.push_back(in.lang_tokens[j]); } else { std::fprintf(stderr, "vla(vla_jepa): lang_tokens has %lld image slots but n_img=%lld\n", (long long) n_img_slots, (long long) n_img); return {}; } @@ -406,13 +464,19 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector image_pos_idx, emb_pos_idx; for (int64_t p = 0; p < SEQ; ++p) { - if (input_ids[p] == (int32_t) image_token_index) image_pos_idx.push_back((int32_t) p); - if (input_ids[p] == (int32_t) embodied_token_id) emb_pos_idx.push_back((int32_t) p); + if (input_ids[p] == (int32_t) image_token_index) + image_pos_idx.push_back((int32_t) p); + if (input_ids[p] == (int32_t) embodied_token_id) + emb_pos_idx.push_back((int32_t) p); } if ((int64_t) emb_pos_idx.size() != num_future) { std::fprintf(stderr, "vla(vla_jepa): found %zu embodied tokens, expected %lld\n", emb_pos_idx.size(), (long long) num_future); return {}; } std::vector> ds_pad(3); - for (int j = 0; j < 3; ++j) { ds_pad[j].assign((size_t) SEQ * H, 0.0f); for (int64_t k = 0; k < n_img; ++k) std::memcpy(ds_pad[j].data() + (size_t) image_pos_idx[k] * H, ds_host[j].data() + (size_t) k * H, H * sizeof(float)); } + for (int j = 0; j < 3; ++j) { + ds_pad[j].assign((size_t) SEQ * H, 0.0f); + for (int64_t k = 0; k < n_img; ++k) + std::memcpy(ds_pad[j].data() + (size_t) image_pos_idx[k] * H, ds_host[j].data() + (size_t) k * H, H * sizeof(float)); + } const LmKey lkey{ SEQ, num_future }; const bool lm_built = lm_graph.ensure(backend, lkey, (size_t) 512 * 1024 * 1024, @@ -422,11 +486,15 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); ggml_tensor * t_emb_idx= ggml_new_tensor_1d(C, GGML_TYPE_I32, num_future); ggml_set_input(t_emb_idx); ggml_tensor * t_ds[3]; - for (int j = 0; j < 3; ++j) { t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_ds[j]); } + for (int j = 0; j < 3; ++j) { + t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); + ggml_set_input(t_ds[j]); + } ggml_tensor * hh = t_embeds; for (int64_t i = 0; i < lm_layers; ++i) { hh = lm.block(C, lm.blk[i], hh, t_pos2, t_lmmask, SEQ); - if (i < 3) hh = ggml_add(C, hh, t_ds[i]); + if (i < 3) + hh = ggml_add(C, hh, t_ds[i]); } ggml_tensor * eagle = hh; ggml_set_output(eagle); @@ -457,11 +525,22 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; - for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { img_start = i; break; } + for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { + img_start = i; + break; + } const int64_t text_end = (img_start < 0) ? SEQ : img_start; const int64_t text_len = text_end - st; - for (int64_t i = 0; i < text_len; ++i) { const int32_t p = (int32_t) (i + st_idx); pp[0*SEQ+(st+i)]=p; pp[1*SEQ+(st+i)]=p; pp[2*SEQ+(st+i)]=p; } - if (img_start < 0) { st_idx += text_len; break; } + for (int64_t i = 0; i < text_len; ++i) { + const int32_t p = (int32_t) (i + st_idx); + pp[0*SEQ+(st+i)]=p; + pp[1*SEQ+(st+i)]=p; + pp[2*SEQ+(st+i)]=p; + } + if (img_start < 0) { + st_idx += text_len; + break; + } int64_t img_end = img_start; while (img_end < SEQ && input_ids[img_end] == (int32_t) image_token_index) ++img_end; const int64_t n_img_tokens = img_end - img_start; const int64_t this_t = n_img_tokens / (llm_grid * llm_grid); @@ -476,21 +555,29 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::memcpy(pp.data() + (size_t) 3 * SEQ, pp.data(), (size_t) SEQ * sizeof(int32_t)); ggml_backend_tensor_set(t_pos2, pp.data(), 0, ggml_nbytes(t_pos2)); } - if (c_mask_seq != SEQ) { build_causal_mask(SEQ, c_mask); c_mask_seq = SEQ; } + if (c_mask_seq != SEQ) { + build_causal_mask(SEQ, c_mask); + c_mask_seq = SEQ; + } ggml_backend_tensor_set(t_lmmask, c_mask.data(), 0, ggml_nbytes(t_lmmask)); ggml_backend_tensor_set(t_emb_idx, emb_pos_idx.data(), 0, ggml_nbytes(t_emb_idx)); - for (int j = 0; j < 3; ++j) ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); + for (int j = 0; j < 3; ++j) + ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); const auto tp0 = std::chrono::steady_clock::now(); if (ggml_backend_graph_compute(backend, lg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): LM compute failed\n"); return {}; } stats.ms_prefill = std::chrono::duration(std::chrono::steady_clock::now() - tp0).count(); - if (dump_prefix) { dump_t("eagle", eagle); dump_t("conditioning", conditioning); } + if (dump_prefix) { + dump_t("eagle", eagle); + dump_t("conditioning", conditioning); + } ggml_backend_tensor_get(conditioning, cond_host.data(), 0, cond_host.size() * sizeof(float)); } // Dumping adds graph outputs, so it always rebuilds. std::vector step_seq, step_pred, step_vel, step_act; - if (dump_prefix) head_graph.release(); + if (dump_prefix) + head_graph.release(); const HeadKey hkey{ num_steps }; const bool head_built = head_graph.ensure(backend, hkey, (size_t) 256 * 1024 * 1024, [&](ggml_context * C, HeadIO & gio) -> ggml_cgraph * { @@ -498,7 +585,12 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, state_dim, 1); ggml_set_input(t_state); ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); ggml_set_input(t_tau[s]); t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, time_proj_dim); ggml_set_input(t_tproj[s]); } + for (int64_t s = 0; s < num_steps; ++s) { + t_tau[s] = ggml_new_tensor_2d(C, GGML_TYPE_F32, E, AH); + ggml_set_input(t_tau[s]); + t_tproj[s] = ggml_new_tensor_1d(C, GGML_TYPE_F32, time_proj_dim); + ggml_set_input(t_tproj[s]); + } ggml_tensor * state_features = ggml_add(C, ggml_mul_mat(C, se_l2W, ggml_relu(C, ggml_add(C, ggml_mul_mat(C, se_l1W, t_state), se_l1b))), se_l2b); ggml_tensor * future = future_tokens; @@ -536,7 +628,12 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { step_vel[s] = vel; actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); step_act[s] = actions; - if (dump_prefix) { ggml_set_output(step_seq[s]); ggml_set_output(step_pred[s]); ggml_set_output(step_vel[s]); ggml_set_output(step_act[s]); } + if (dump_prefix) { + ggml_set_output(step_seq[s]); + ggml_set_output(step_pred[s]); + ggml_set_output(step_vel[s]); + ggml_set_output(step_act[s]); + } } ggml_set_output(actions); gio.t_cond=t_cond; gio.t_state=t_state; gio.t_x0=t_x0; gio.actions=actions; @@ -544,7 +641,12 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_cgraph * hg = ggml_new_graph_custom(C, 65536, false); ggml_build_forward_expand(hg, actions); - if (dump_prefix) for (int64_t s = 0; s < num_steps; ++s) { ggml_build_forward_expand(hg, step_seq[s]); ggml_build_forward_expand(hg, step_pred[s]); ggml_build_forward_expand(hg, step_vel[s]); ggml_build_forward_expand(hg, step_act[s]); } + if (dump_prefix) for (int64_t s = 0; s < num_steps; ++s) { + ggml_build_forward_expand(hg, step_seq[s]); + ggml_build_forward_expand(hg, step_pred[s]); + ggml_build_forward_expand(hg, step_vel[s]); + ggml_build_forward_expand(hg, step_act[s]); + } return hg; }); if (!head_built) { std::fprintf(stderr, "vla(vla_jepa): head graph build failed\n"); return {}; } @@ -555,9 +657,17 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector & t_tau = hio.t_tau; std::vector & t_tproj = hio.t_tproj; ggml_backend_tensor_set(t_cond, cond_host.data(), 0, ggml_nbytes(t_cond)); - { std::vector st(state_dim, 0.0f); for (int64_t i = 0; i < state_dim; ++i) st[i] = in.state ? in.state[i] : 0.0f; ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); } + { + std::vector st(state_dim, 0.0f); + for (int64_t i = 0; i < state_dim; ++i) + st[i] = in.state ? in.state[i] : 0.0f; + ggml_backend_tensor_set(t_state, st.data(), 0, ggml_nbytes(t_state)); + } ggml_backend_tensor_set(t_x0, x_init.data(), 0, ggml_nbytes(t_x0)); - for (int64_t s = 0; s < num_steps; ++s) { ggml_backend_tensor_set(t_tau[s], c_tau[(size_t) s].data(), 0, ggml_nbytes(t_tau[s])); ggml_backend_tensor_set(t_tproj[s], c_tproj[(size_t) s].data(), 0, ggml_nbytes(t_tproj[s])); } + for (int64_t s = 0; s < num_steps; ++s) { + ggml_backend_tensor_set(t_tau[s], c_tau[(size_t) s].data(), 0, ggml_nbytes(t_tau[s])); + ggml_backend_tensor_set(t_tproj[s], c_tproj[(size_t) s].data(), 0, ggml_nbytes(t_tproj[s])); + } const auto td0 = std::chrono::steady_clock::now(); if (ggml_backend_graph_compute(backend, hg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): head compute failed\n"); return {}; } diff --git a/src/modules/dit_head.cpp b/src/modules/dit_head.cpp index 48f6942..f081288 100644 --- a/src/modules/dit_head.cpp +++ b/src/modules/dit_head.cpp @@ -25,7 +25,8 @@ namespace vla { void DitHead::declare(WeightLoader & L, const char * prefix, bool fuse_qkv, bool interleave, const char * outer) { - if (!outer) outer = prefix; + if (!outer) + outer = prefix; te_l1W = L.gemm("%s.time_emb.l1.weight", outer); te_l1b = L.f32 ("%s.time_emb.l1.bias", outer); @@ -116,8 +117,13 @@ ggml_tensor * DitHead::block(ggml_context * C, const DitLayerW & w, ggml_tensor V = ggml_cont(C, ggml_permute(C, head_view(C, qkv, hd, heads, Tk, dim, 3, 2), 1, 2, 0, 3)); } else { Q = to_heads(C, linear(C, w.Wq, w.bq, n), hd, heads, Tk); - if (K_pre) { K = K_pre; V = V_pre; } - else { kv(C, w, enc ? enc : n, &K, &V); } + if (K_pre) { + K = K_pre; + V = V_pre; + } + else { + kv(C, w, enc ? enc : n, &K, &V); + } } ggml_tensor * att = attention(C, Q, K, V, nullptr, scale, dim, Tk); diff --git a/src/modules/dual_tower.h b/src/modules/dual_tower.h index edfcf80..ffee524 100644 --- a/src/modules/dual_tower.h +++ b/src/modules/dual_tower.h @@ -78,7 +78,9 @@ struct DualTower { } }; -inline ggml_tensor * LN(ggml_context*C, ggml_tensor*x, ggml_tensor*w, ggml_tensor*b, float eps){ return ggml_add(C,ggml_mul(C,ggml_norm(C,x,eps),w),b); } +inline ggml_tensor * LN(ggml_context*C, ggml_tensor*x, ggml_tensor*w, ggml_tensor*b, float eps){ + return ggml_add(C,ggml_mul(C,ggml_norm(C,x,eps),w),b); +} inline ggml_tensor* vit_block(ggml_context*C, const ViTLayerW&w, ggml_tensor*x, int64_t N, int64_t hidden, int64_t heads, int64_t hd, float eps, bool ls){ const float sc=1.0f/std::sqrt((float)hd); @@ -113,9 +115,14 @@ inline ggml_tensor* tower(ggml_context*C, ggml_tensor*pix, ggml_tensor*pw, ggml_ ggml_tensor*pt=ggml_cont(C,ggml_transpose(C,ggml_reshape_2d(C,conv,NP,hidden))); pt=ggml_add(C,pt,pb); pt=ggml_add(C,pt,pos); ggml_tensor*x=pt; - if(prefix){ ggml_tensor*tok=ggml_concat(C,ggml_reshape_2d(C,cls,hidden,1),reg,1); x=ggml_concat(C,tok,pt,1); } - for(size_t i=0;inb[1],nprefix*x->nb[1])); + if(prefix){ + ggml_tensor*tok=ggml_concat(C,ggml_reshape_2d(C,cls,hidden,1),reg,1); + x=ggml_concat(C,tok,pt,1); + } + for(size_t i=0;inb[1],nprefix*x->nb[1])); return x; } diff --git a/src/modules/encoder.cpp b/src/modules/encoder.cpp index 6c31ac1..f9dc636 100644 --- a/src/modules/encoder.cpp +++ b/src/modules/encoder.cpp @@ -61,8 +61,10 @@ ggml_tensor * EncStack::block(ggml_context * C, const EncBlockW & w, ggml_tensor ggml_tensor * K = to_heads(C, k, hd, heads, seq, nv); ggml_tensor * att; - if (cfg.flash_attn) att = flash_attention(C, Q, K, to_heads(C, v, hd, heads, seq, nv), nullptr, scale); - else att = attention(C, Q, K, to_heads_v(C, v, hd, heads, seq, nv), nullptr, scale, cfg.hidden, seq, nv); + if (cfg.flash_attn) + att = flash_attention(C, Q, K, to_heads(C, v, hd, heads, seq, nv), nullptr, scale); + else + att = attention(C, Q, K, to_heads_v(C, v, hd, heads, seq, nv), nullptr, scale, cfg.hidden, seq, nv); ggml_tensor * h1 = ggml_add(C, x, linear(C, w.Wo, w.bo, att)); ggml_tensor * n2 = layer_norm(C, h1, w.ln2w, w.ln2b, cfg.ln_eps); diff --git a/src/modules/gemma_expert.h b/src/modules/gemma_expert.h index 1faf840..33e1001 100644 --- a/src/modules/gemma_expert.h +++ b/src/modules/gemma_expert.h @@ -57,7 +57,8 @@ struct GemmaStack { w.Wup = L.gemm ("%s.blk.%lld.ffn_up.weight", prefix, (long long)i); w.Wdown = L.gemm ("%s.blk.%lld.ffn_down.weight", prefix, (long long)i); } - if (with_output_norm) output_norm = L.f32_gemma_norm("%s.output_norm.weight", prefix); + if (with_output_norm) + output_norm = L.f32_gemma_norm("%s.output_norm.weight", prefix); } }; diff --git a/src/modules/preprocess.h b/src/modules/preprocess.h index 203a97f..b91ec7a 100644 --- a/src/modules/preprocess.h +++ b/src/modules/preprocess.h @@ -65,8 +65,10 @@ inline bool preprocess_image_chw(const char * arch, const ImageView & v, int64_t for (int64_t w = 0; w < side; ++w) for (int64_t c = 0; c < 3; ++c) { float px; - if (v.format == PixelFormat::U8) px = ((const uint8_t *) v.data)[(h * side + w) * 3 + c] / 255.0f; - else px = ((const float *) v.data)[(h * side + w) * 3 + c]; + if (v.format == PixelFormat::U8) + px = ((const uint8_t *) v.data)[(h * side + w) * 3 + c] / 255.0f; + else + px = ((const float *) v.data)[(h * side + w) * 3 + c]; out[c * side * side + h * side + w] = px * 2.0f - 1.0f; } return true; @@ -83,7 +85,8 @@ inline bool preprocess_image_patches(const char * arch, const ImageView & v, int out.assign((size_t) pd*np, 0.0f); auto px = [&](int64_t r, int64_t c, int64_t ch) -> float { - if (v.format == PixelFormat::U8) return ((const uint8_t *) v.data)[(r*side+c)*3+ch]/255.0f; + if (v.format == PixelFormat::U8) + return ((const uint8_t *) v.data)[(r*side+c)*3+ch]/255.0f; return ((const float *) v.data)[(r*side+c)*3+ch]; }; diff --git a/src/modules/prompt.cpp b/src/modules/prompt.cpp index 6352627..7b1c725 100644 --- a/src/modules/prompt.cpp +++ b/src/modules/prompt.cpp @@ -27,14 +27,17 @@ bool build_prompt(const char * arch, const Inputs & in, int64_t n_img, int64_t slots = 0; for (int j = 0; j < in.n_lang; ++j) - if (in.lang_tokens[j] == image_token) ++slots; + if (in.lang_tokens[j] == image_token) + ++slots; if (slots == n_img) { out.ids.assign(in.lang_tokens, in.lang_tokens+in.n_lang); } else if (slots == 0) { out.ids.reserve((size_t)(n_img+in.n_lang)); - for (int64_t i = 0; i < n_img; ++i) out.ids.push_back(image_token); - for (int j = 0; j < in.n_lang; ++j) out.ids.push_back(in.lang_tokens[j]); + for (int64_t i = 0; i < n_img; ++i) + out.ids.push_back(image_token); + for (int j = 0; j < in.n_lang; ++j) + out.ids.push_back(in.lang_tokens[j]); } else { std::fprintf(stderr, "vla(%s): lang_tokens has %lld image-token slots but n_img=%lld; expected 0 or %lld\n", arch, (long long) slots, (long long) n_img, (long long) n_img); @@ -50,8 +53,10 @@ bool build_prompt(const char * arch, const Inputs & in, int64_t n_img, out.image_pos.reserve((size_t) n_img); out.text_pos.reserve((size_t)(seq-n_img)); for (int64_t p = 0; p < seq; ++p) { - if (out.ids[p] == image_token) out.image_pos.push_back((int32_t) p); - else out.text_pos.push_back((int32_t) p); + if (out.ids[p] == image_token) + out.image_pos.push_back((int32_t) p); + else + out.text_pos.push_back((int32_t) p); } return true; } @@ -60,7 +65,8 @@ bool fetch_embeds(const char * arch, gguf_reader & io, const Prompt & p, const float * img_emb, int64_t hidden, std::vector & out) { const int64_t seq = p.len(); out.assign((size_t) seq*hidden, 0.0f); - if (!io.fetch_rows_f32("token_embd.weight", p.ids, out.data(), hidden)) return false; + if (!io.fetch_rows_f32("token_embd.weight", p.ids, out.data(), hidden)) + return false; for (size_t k = 0; k < p.image_pos.size(); ++k) std::memcpy(out.data()+(size_t) p.image_pos[k]*hidden, img_emb+k*hidden, hidden*sizeof(float)); @@ -78,7 +84,8 @@ void init_noise(const Inputs & in, size_t n, std::vector & out) { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); - for (auto & v : out) v = nd(rng); + for (auto & v : out) + v = nd(rng); } } diff --git a/src/modules/prompt.h b/src/modules/prompt.h index 97c5168..7808190 100644 --- a/src/modules/prompt.h +++ b/src/modules/prompt.h @@ -30,8 +30,12 @@ struct Prompt { std::vector image_pos; std::vector text_pos; - int64_t len() const { return (int64_t) ids.size(); } - int64_t n_text() const { return (int64_t) text_pos.size(); } + int64_t len() const { + return (int64_t) ids.size(); + } + int64_t n_text() const { + return (int64_t) text_pos.size(); + } }; // Accepts a stream carrying exactly n_img placeholders, or none, in which case diff --git a/src/modules/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h index c873a3b..a6233ac 100644 --- a/src/modules/qwen3vl_vit.h +++ b/src/modules/qwen3vl_vit.h @@ -100,8 +100,10 @@ inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml ggml_tensor * Q = rope_2d(C, to_heads(C, q, hd, heads, seq), cos_t, sin_t); ggml_tensor * K = rope_2d(C, to_heads(C, k, hd, heads, seq), cos_t, sin_t); ggml_tensor * att; - if (vla::flash_attn_enabled()) att = flash_attention(C, Q, K, to_heads (C, v, hd, heads, seq), nullptr, scale); - else att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); + if (vla::flash_attn_enabled()) + att = flash_attention(C, Q, K, to_heads (C, v, hd, heads, seq), nullptr, scale); + else + att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); @@ -138,13 +140,21 @@ inline void vit_rope_tables(const std::vector & row, const std::vector< std::vector & cos_t, std::vector & sin_t) { const int64_t S = (int64_t) row.size(), nf = hd / 4; std::vector invf(nf); - for (int64_t i = 0; i < nf; ++i) invf[i] = 1.0 / std::pow(theta, (double)(2 * i) / (double)(hd / 2)); + for (int64_t i = 0; i < nf; ++i) + invf[i] = 1.0 / std::pow(theta, (double)(2 * i) / (double)(hd / 2)); cos_t.assign((size_t) S * hd, 0.0f); sin_t.assign((size_t) S * hd, 0.0f); for (int64_t s = 0; s < S; ++s) { std::vector emb(hd); - for (int64_t i = 0; i < nf; ++i) { emb[i] = (double) row[s] * invf[i]; emb[nf + i] = (double) col[s] * invf[i]; } - for (int64_t i = 0; i < hd / 2; ++i) emb[hd / 2 + i] = emb[i]; - for (int64_t i = 0; i < hd; ++i) { cos_t[s * hd + i] = (float) std::cos(emb[i]); sin_t[s * hd + i] = (float) std::sin(emb[i]); } + for (int64_t i = 0; i < nf; ++i) { + emb[i] = (double) row[s] * invf[i]; + emb[nf + i] = (double) col[s] * invf[i]; + } + for (int64_t i = 0; i < hd / 2; ++i) + emb[hd / 2 + i] = emb[i]; + for (int64_t i = 0; i < hd; ++i) { + cos_t[s * hd + i] = (float) std::cos(emb[i]); + sin_t[s * hd + i] = (float) std::sin(emb[i]); + } } } @@ -166,7 +176,8 @@ inline void interp_pos_embed(const std::vector & table, int64_t num_side, const double c00 = (1 - dh) * (1 - dw), c01 = (1 - dh) * dw, c10 = dh * (1 - dw), c11 = dh * dw; const float * T00 = &table[(h0 * num_side + w0) * hidden]; const float * T01 = &table[(h0 * num_side + w1) * hidden]; const float * T10 = &table[(h1 * num_side + w0) * hidden]; const float * T11 = &table[(h1 * num_side + w1) * hidden]; - for (int64_t c = 0; c < hidden; ++c) out[s * hidden + c] = (float)(c00 * T00[c] + c01 * T01[c] + c10 * T10[c] + c11 * T11[c]); + for (int64_t c = 0; c < hidden; ++c) + out[s * hidden + c] = (float)(c00 * T00[c] + c01 * T01[c] + c10 * T10[c] + c11 * T11[c]); } } @@ -182,7 +193,8 @@ inline bool preprocess_image_patches(const char * arch, const ImageView & v, int const int64_t S = (int64_t) row.size(), pf = 3 * tps * ps * ps; out.assign((size_t) pf * S, 0.0f); auto px = [&](int64_t r, int64_t c, int64_t ch) -> float { - if (v.format == PixelFormat::U8) return ((const uint8_t *) v.data)[(r * side + c) * 3 + ch] / 255.0f; + if (v.format == PixelFormat::U8) + return ((const uint8_t *) v.data)[(r * side + c) * 3 + ch] / 255.0f; return ((const float *) v.data)[(r * side + c) * 3 + ch]; }; for (int64_t s = 0; s < S; ++s) @@ -190,7 +202,8 @@ inline bool preprocess_image_patches(const char * arch, const ImageView & v, int for (int64_t ph = 0; ph < ps; ++ph) for (int64_t pw = 0; pw < ps; ++pw) { const float val = (px(row[s] * ps + ph, col[s] * ps + pw, ch) - QWEN3VL_MEAN[ch]) / QWEN3VL_STD[ch]; - for (int64_t t = 0; t < tps; ++t) out[s * pf + ch * tps * ps * ps + t * ps * ps + ph * ps + pw] = val; + for (int64_t t = 0; t < tps; ++t) + out[s * pf + ch * tps * ps * ps + t * ps * ps + ph * ps + pw] = val; } return true; } diff --git a/src/options.cpp b/src/options.cpp index b4fa63f..b9ee2fd 100644 --- a/src/options.cpp +++ b/src/options.cpp @@ -26,21 +26,34 @@ namespace vla { namespace { bool parse_dtype(const std::string & v, ggml_type & out) { - if (v == "f32" || v == "fp32") { out = GGML_TYPE_F32; return true; } - if (v == "bf16" || v == "bfp16") { out = GGML_TYPE_BF16; return true; } + if (v == "f32" || v == "fp32") { + out = GGML_TYPE_F32; + return true; + } + if (v == "bf16" || v == "bfp16") { + out = GGML_TYPE_BF16; + return true; + } return false; } bool parse_bool(const std::string & v, bool & out) { - if (v == "1" || v == "true" || v == "on" || v == "yes") { out = true; return true; } - if (v == "0" || v == "false" || v == "off" || v == "no") { out = false; return true; } + if (v == "1" || v == "true" || v == "on" || v == "yes") { + out = true; + return true; + } + if (v == "0" || v == "false" || v == "off" || v == "no") { + out = false; + return true; + } return false; } bool parse_int(const std::string & v, int & out) { char * end = nullptr; const long n = std::strtol(v.c_str(), &end, 10); - if (end == v.c_str() || *end) return false; + if (end == v.c_str() || *end) + return false; out = (int) n; return true; } @@ -56,11 +69,19 @@ bool g_flash_attn = false; bool g_mm_prec_f32 = true; } -void set_flash_attn(bool on) { g_flash_attn = on; } -bool flash_attn_enabled() { return g_flash_attn; } +void set_flash_attn(bool on) { + g_flash_attn = on; +} +bool flash_attn_enabled() { + return g_flash_attn; +} -void set_mm_prec_f32(bool on) { g_mm_prec_f32 = on; } -bool mm_prec_f32_enabled() { return g_mm_prec_f32; } +void set_mm_prec_f32(bool on) { + g_mm_prec_f32 = on; +} +bool mm_prec_f32_enabled() { + return g_mm_prec_f32; +} const char * Options::usage() { return " --weight-dtype f32|bf16 resident dtype for GEMM weights\n" @@ -77,51 +98,86 @@ bool Options::parse_arg(int argc, char ** argv, int & i, std::string & err) { const std::string a = argv[i]; auto next = [&](std::string & v) -> bool { - if (i+1 >= argc) { err = a+" needs a value"; return false; } + if (i+1 >= argc) { + err = a+" needs a value"; + return false; + } v = argv[++i]; return true; }; if (a == "--weight-dtype" || a == "--act-dtype") { std::string v; - if (!next(v)) return false; + if (!next(v)) + return false; ggml_type t; - if (!parse_dtype(v, t)) { err = a+": expected f32 or bf16, got '"+v+"'"; return false; } - if (a == "--weight-dtype") weight_dtype = t; - else act_dtype = t; + if (!parse_dtype(v, t)) { + err = a+": expected f32 or bf16, got '"+v+"'"; + return false; + } + if (a == "--weight-dtype") + weight_dtype = t; + else + act_dtype = t; return true; } if (a == "--flash-attn") { bool v = true; - if (i+1 < argc && argv[i+1][0] != '-' && parse_bool(argv[i+1], v)) ++i; + if (i+1 < argc && argv[i+1][0] != '-' && parse_bool(argv[i+1], v)) + ++i; flash_attn = v; return true; } if (a == "--mm-prec") { std::string v; - if (!next(v)) return false; - if (v == "default") { mm_prec_f32 = false; return true; } - if (v == "f32") { mm_prec_f32 = true; return true; } + if (!next(v)) + return false; + if (v == "default") { + mm_prec_f32 = false; + return true; + } + if (v == "f32") { + mm_prec_f32 = true; + return true; + } err = "--mm-prec: expected default or f32, got '"+v+"'"; return false; } if (a == "--n-threads" || a == "--num-steps") { std::string v; - if (!next(v)) return false; + if (!next(v)) + return false; int n = 0; - if (!parse_int(v, n) || n <= 0) { err = a+": expected a positive integer, got '"+v+"'"; return false; } - if (a == "--n-threads") n_threads = n; - else num_steps = n; + if (!parse_int(v, n) || n <= 0) { + err = a+": expected a positive integer, got '"+v+"'"; + return false; + } + if (a == "--n-threads") + n_threads = n; + else + num_steps = n; return true; } - if (a == "--embodiment") { std::string v; if (!next(v)) return false; embodiment = v; return true; } - if (a == "--unnorm-key") { std::string v; if (!next(v)) return false; unnorm_key = v; return true; } + if (a == "--embodiment") { + std::string v; + if (!next(v)) + return false; + embodiment = v; + return true; + } + if (a == "--unnorm-key") { + std::string v; + if (!next(v)) + return false; + unnorm_key = v; + return true; + } err.clear(); return false; @@ -158,10 +214,12 @@ bool Options::reject_retired_env(std::string & err) { } bool Options::load_json(const std::string & path, std::string & err) { - if (path.empty()) return true; + if (path.empty()) + return true; std::ifstream f(path); - if (!f) return true; + if (!f) + return true; nlohmann::json j; try { @@ -170,19 +228,28 @@ bool Options::load_json(const std::string & path, std::string & err) { err = std::string("config json: ")+e.what(); return false; } - if (!j.contains("runtime") || !j["runtime"].is_object()) return true; + if (!j.contains("runtime") || !j["runtime"].is_object()) + return true; const nlohmann::json & r = j["runtime"]; try { ggml_type t; - if (r.contains("weight_dtype") && parse_dtype(r["weight_dtype"].get(), t)) weight_dtype = t; - if (r.contains("act_dtype") && parse_dtype(r["act_dtype"].get(), t)) act_dtype = t; - if (r.contains("flash_attn")) flash_attn = r["flash_attn"].get(); - if (r.contains("mm_prec")) mm_prec_f32 = r["mm_prec"].get() == "f32"; - if (r.contains("n_threads")) n_threads = r["n_threads"].get(); - if (r.contains("num_steps")) num_steps = r["num_steps"].get(); - if (r.contains("embodiment")) embodiment = r["embodiment"].get(); - if (r.contains("unnorm_key")) unnorm_key = r["unnorm_key"].get(); + if (r.contains("weight_dtype") && parse_dtype(r["weight_dtype"].get(), t)) + weight_dtype = t; + if (r.contains("act_dtype") && parse_dtype(r["act_dtype"].get(), t)) + act_dtype = t; + if (r.contains("flash_attn")) + flash_attn = r["flash_attn"].get(); + if (r.contains("mm_prec")) + mm_prec_f32 = r["mm_prec"].get() == "f32"; + if (r.contains("n_threads")) + n_threads = r["n_threads"].get(); + if (r.contains("num_steps")) + num_steps = r["num_steps"].get(); + if (r.contains("embodiment")) + embodiment = r["embodiment"].get(); + if (r.contains("unnorm_key")) + unnorm_key = r["unnorm_key"].get(); } catch (const std::exception & e) { err = std::string("config json runtime: ")+e.what(); return false; diff --git a/src/scratch_ctx.h b/src/scratch_ctx.h index 9c11650..8db90ae 100644 --- a/src/scratch_ctx.h +++ b/src/scratch_ctx.h @@ -36,23 +36,35 @@ class scratch_ctx { scratch_ctx() = default; scratch_ctx(const scratch_ctx &) = delete; scratch_ctx & operator=(const scratch_ctx &) = delete; - ~scratch_ctx() { release(); } + ~scratch_ctx() { + release(); + } ggml_context * reset(size_t arena) { - if (ctx_) { ggml_reset(ctx_); return ctx_; } + if (ctx_) { + ggml_reset(ctx_); + return ctx_; + } ggml_init_params p = { arena, nullptr, true }; ctx_ = ggml_init(p); return ctx_; } bool alloc(ggml_backend_t backend, ggml_cgraph * gf) { - if (!galloc_) galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (!galloc_) + galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); return galloc_ && ggml_gallocr_alloc_graph(galloc_, gf); } void release() { - if (galloc_) { ggml_gallocr_free(galloc_); galloc_ = nullptr; } - if (ctx_) { ggml_free(ctx_); ctx_ = nullptr; } + if (galloc_) { + ggml_gallocr_free(galloc_); + galloc_ = nullptr; + } + if (ctx_) { + ggml_free(ctx_); + ctx_ = nullptr; + } } private: @@ -69,31 +81,51 @@ class graph_cache { graph_cache() = default; graph_cache(const graph_cache &) = delete; graph_cache & operator=(const graph_cache &) = delete; - ~graph_cache() { release(); } + ~graph_cache() { + release(); + } template bool ensure(ggml_backend_t backend, const Key & key, size_t arena, Build && build) { - if (valid_ && key_ == key) return true; + if (valid_ && key_ == key) + return true; release(); ggml_init_params p = { arena, nullptr, true }; ctx_ = ggml_init(p); - if (!ctx_) return false; + if (!ctx_) + return false; io_ = IO{}; gf_ = build(ctx_, io_); - if (!gf_) { release(); return false; } + if (!gf_) { + release(); + return false; + } galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!galloc_ || !ggml_gallocr_alloc_graph(galloc_, gf_)) { release(); return false; } + if (!galloc_ || !ggml_gallocr_alloc_graph(galloc_, gf_)) { + release(); + return false; + } key_ = key; valid_ = true; return true; } - IO & io() { return io_; } - ggml_cgraph * graph() { return gf_; } + IO & io() { + return io_; + } + ggml_cgraph * graph() { + return gf_; + } void release() { - if (galloc_) { ggml_gallocr_free(galloc_); galloc_ = nullptr; } - if (ctx_) { ggml_free(ctx_); ctx_ = nullptr; } + if (galloc_) { + ggml_gallocr_free(galloc_); + galloc_ = nullptr; + } + if (ctx_) { + ggml_free(ctx_); + ctx_ = nullptr; + } gf_ = nullptr; io_ = IO{}; valid_ = false; diff --git a/src/serving/hf_fetch.h b/src/serving/hf_fetch.h index cfe6c33..69c664b 100644 --- a/src/serving/hf_fetch.h +++ b/src/serving/hf_fetch.h @@ -26,22 +26,27 @@ namespace vla { // Repo ids reach a shell command, so reject anything outside this set. inline bool hf_token_ok(const std::string & s, bool allow_slash) { - if (s.empty() || s.size() > 200) return false; + if (s.empty() || s.size() > 200) + return false; // A leading '/' would make fs::path join replace the cache root instead of // extending it, putting the download anywhere on disk. - if (s.front() == '-' || s.front() == '/' || s.find("..") != std::string::npos) return false; + if (s.front() == '-' || s.front() == '/' || s.find("..") != std::string::npos) + return false; for (const char c : s) { const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-' || (allow_slash && c == '/'); - if (!ok) return false; + if (!ok) + return false; } return true; } inline std::string hf_cache_root() { - if (const char * e = std::getenv("VLA_CACHE"); e && *e) return e; - if (const char * h = std::getenv("HOME"); h && *h) return std::string(h) + "/.cache/vla"; + if (const char * e = std::getenv("VLA_CACHE"); e && *e) + return e; + if (const char * h = std::getenv("HOME"); h && *h) + return std::string(h) + "/.cache/vla"; return ".vla-cache"; } @@ -52,16 +57,23 @@ inline std::string hf_pick_gguf(const std::filesystem::path & dir, const std::st std::string best; uintmax_t best_size = 0; for (fs::recursive_directory_iterator it(dir, ec), end; it != end && !ec; it.increment(ec)) { - if (!it->is_regular_file(ec) || it->path().extension() != ".gguf") continue; + if (!it->is_regular_file(ec) || it->path().extension() != ".gguf") + continue; const std::string name = it->path().filename().string(); if (!want.empty()) { - if (name == want) return it->path().string(); + if (name == want) + return it->path().string(); continue; } - if (name.rfind("mmproj", 0) == 0) continue; + if (name.rfind("mmproj", 0) == 0) + continue; const uintmax_t sz = it->file_size(ec); - if (ec) continue; - if (sz > best_size) { best_size = sz; best = it->path().string(); } + if (ec) + continue; + if (sz > best_size) { + best_size = sz; + best = it->path().string(); + } } return best; } @@ -85,7 +97,8 @@ inline std::string hf_resolve(const std::string & spec) { std::error_code ec; if (fs::is_directory(dir, ec)) { const std::string hit = hf_pick_gguf(dir, file); - if (!hit.empty()) return hit; + if (!hit.empty()) + return hit; } fs::create_directories(dir, ec); @@ -102,7 +115,8 @@ inline std::string hf_resolve(const std::string & spec) { } std::string cmd = "hf download " + repo; - if (!file.empty()) cmd += " " + file; + if (!file.empty()) + cmd += " " + file; cmd += " --local-dir '" + dir.string() + "'"; std::fprintf(stderr, "vla: %s\n", cmd.c_str()); @@ -115,7 +129,8 @@ inline std::string hf_resolve(const std::string & spec) { } const std::string hit = hf_pick_gguf(dir, file); - if (hit.empty()) std::fprintf(stderr, "vla: no .gguf under %s after download\n", dir.string().c_str()); + if (hit.empty()) + std::fprintf(stderr, "vla: no .gguf under %s after download\n", dir.string().c_str()); return hit; } diff --git a/src/serving/server.cpp b/src/serving/server.cpp index 26790a0..2ba432c 100644 --- a/src/serving/server.cpp +++ b/src/serving/server.cpp @@ -41,7 +41,9 @@ namespace { std::atomic g_shutdown{false}; -void on_signal(int) { g_shutdown.store(true, std::memory_order_relaxed); } +void on_signal(int) { + g_shutdown.store(true, std::memory_order_relaxed); +} // Reject absurd image dimensions before any size arithmetic, so an untrusted // width or height cannot overflow size_t or truncate to a negative int. @@ -160,7 +162,8 @@ Drain drain_extra_frames(zmq::socket_t & sock) { Drain d = Drain::Clean; while (sock.get(zmq::sockopt::rcvmore)) { zmq::message_t junk; - if (!sock.recv(junk, zmq::recv_flags::none)) return Drain::Stalled; + if (!sock.recv(junk, zmq::recv_flags::none)) + return Drain::Stalled; d = Drain::Extra; } return d; @@ -168,7 +171,8 @@ Drain drain_extra_frames(zmq::socket_t & sock) { int find_non_finite(const float * data, int n) { for (int i = 0; i < n; ++i) { - if (!std::isfinite(data[i])) return i; + if (!std::isfinite(data[i])) + return i; } return -1; } @@ -225,7 +229,8 @@ int main(int argc, char ** argv) { config_path = argv[++i]; } else if (a == "--timing-detail" && i + 1 < argc) { const std::string v = argv[++i]; - if (v == "none") timing_detail = vla::TimingDetail::NONE; + if (v == "none") + timing_detail = vla::TimingDetail::NONE; else if (v == "phase") timing_detail = vla::TimingDetail::PHASE; else { std::fprintf(stderr, "vla-server: bad --timing-detail value '%s'\n", v.c_str()); @@ -247,7 +252,8 @@ int main(int argc, char ** argv) { } if (!hf_spec.empty() && positionals.empty()) { ckpt_path = vla::hf_resolve(hf_spec); - if (ckpt_path.empty()) return 1; + if (ckpt_path.empty()) + return 1; } else if (positionals.size() == 1) { ckpt_path = positionals[0]; } else if (positionals.size() == 2) { @@ -332,20 +338,26 @@ int main(int argc, char ** argv) { try { zmq::poll(poll, 1, std::chrono::milliseconds(200)); } catch (const zmq::error_t & e) { - if (e.num() == EINTR) continue; - if (e.num() == ETERM) break; + if (e.num() == EINTR) + continue; + if (e.num() == ETERM) + break; std::fprintf(stderr, "vla-server: zmq error: %s\n", e.what()); continue; } - if (!(poll[0].revents & ZMQ_POLLIN)) continue; + if (!(poll[0].revents & ZMQ_POLLIN)) + continue; zmq::message_t req_msg; try { auto rr = sock.recv(req_msg, zmq::recv_flags::none); - if (!rr) continue; + if (!rr) + continue; } catch (const zmq::error_t & e) { - if (e.num() == EINTR) continue; - if (e.num() == ETERM) break; + if (e.num() == EINTR) + continue; + if (e.num() == ETERM) + break; std::fprintf(stderr, "vla-server: zmq error: %s\n", e.what()); continue; } @@ -402,7 +414,8 @@ int main(int argc, char ** argv) { break; } } - if (!tokens_ok) continue; + if (!tokens_ok) + continue; } if (req.state_size() != int(cfg.max_state_dim)) { char buf[128]; std::snprintf(buf, sizeof(buf), @@ -464,7 +477,8 @@ int main(int argc, char ** argv) { break; } } - if (!decode_ok) continue; + if (!decode_ok) + continue; } std::vector lang_tokens(req.lang_tokens().begin(), req.lang_tokens().end()); @@ -526,7 +540,8 @@ int main(int argc, char ** argv) { vla::PredictResponse resp; resp.set_request_id(rid); resp.mutable_action_chunk()->Reserve(static_cast(action_chunk.size())); - for (float v : action_chunk) resp.add_action_chunk(v); + for (float v : action_chunk) + resp.add_action_chunk(v); resp.set_chunk_size(static_cast(cfg.n_suffix)); resp.set_action_dim(static_cast(cfg.max_action_dim)); resp.set_latency_ms_total(st.ms_total); diff --git a/src/serving/vla-bench.cpp b/src/serving/vla-bench.cpp index 1a070eb..d51b208 100644 --- a/src/serving/vla-bench.cpp +++ b/src/serving/vla-bench.cpp @@ -47,7 +47,8 @@ void usage(const char * prog) { // v must be sorted. double percentile(const std::vector & v, double p) { - if (v.empty()) return 0.0; + if (v.empty()) + return 0.0; const double idx = p * (double) (v.size() - 1); const size_t lo = (size_t) std::floor(idx), hi = (size_t) std::ceil(idx); return v[lo] + (v[hi] - v[lo]) * (idx - (double) lo); @@ -64,10 +65,14 @@ int main(int argc, char ** argv) { for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto need = [&](const char * name) -> const char * { - if (i + 1 >= argc) { std::fprintf(stderr, "vla-bench: %s needs a value\n", name); std::exit(1); } + if (i + 1 >= argc) { + std::fprintf(stderr, "vla-bench: %s needs a value\n", name); + std::exit(1); + } return argv[++i]; }; - if (a == "--ckpt") ckpt = need("--ckpt"); + if (a == "--ckpt") + ckpt = need("--ckpt"); else if (a == "-hf") hf = need("-hf"); else if (a == "--mmproj") mmproj = need("--mmproj"); else if (a == "--label") label = need("--label"); @@ -79,16 +84,30 @@ int main(int argc, char ** argv) { else if (a == "--warmup") warmup = std::atoi(need("--warmup")); else if (a == "--reps") reps = std::atoi(need("--reps")); else if (a == "--markdown") markdown = true; - else if (a == "-h" || a == "--help") { usage(argv[0]); return 0; } - else { std::fprintf(stderr, "vla-bench: unknown argument %s\n", a.c_str()); usage(argv[0]); return 1; } + else if (a == "-h" || a == "--help") { + usage(argv[0]); + return 0; + } + else { + std::fprintf(stderr, "vla-bench: unknown argument %s\n", a.c_str()); + usage(argv[0]); + return 1; + } } if (!hf.empty()) { - if (!ckpt.empty()) { std::fprintf(stderr, "vla-bench: pass --ckpt or -hf, not both\n"); return 1; } + if (!ckpt.empty()) { + std::fprintf(stderr, "vla-bench: pass --ckpt or -hf, not both\n"); + return 1; + } ckpt = vla::hf_resolve(hf); - if (ckpt.empty()) return 1; + if (ckpt.empty()) + return 1; + } + if (ckpt.empty()) { + usage(argv[0]); + return 1; } - if (ckpt.empty()) { usage(argv[0]); return 1; } if (n_images < 1 || side < 16 || n_tokens < 1 || warmup < 0 || reps < 1) { std::fprintf(stderr, "vla-bench: --images/--size/--tokens/--reps must be positive\n"); return 1; @@ -99,7 +118,10 @@ int main(int argc, char ** argv) { } vla::Model * m = vla::model_load(mmproj, ckpt, ""); - if (!m) { std::fprintf(stderr, "vla-bench: model_load failed\n"); return 1; } + if (!m) { + std::fprintf(stderr, "vla-bench: model_load failed\n"); + return 1; + } const vla::Config & cfg = vla::model_config(m); std::vector> pixels(n_images, std::vector((size_t) 3 * side * side)); @@ -113,14 +135,18 @@ int main(int argc, char ** argv) { } std::vector lang((size_t) n_tokens); - for (int i = 0; i < n_tokens; ++i) lang[i] = 1 + (i % 100); - if (extra_token >= 0 && extra_count > 0) lang.insert(lang.end(), (size_t) extra_count, extra_token); + for (int i = 0; i < n_tokens; ++i) + lang[i] = 1 + (i % 100); + if (extra_token >= 0 && extra_count > 0) + lang.insert(lang.end(), (size_t) extra_count, extra_token); std::vector state((size_t) cfg.max_state_dim, 0.0f); - for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) state[i] = 0.01f * (float) (i + 1); + for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) + state[i] = 0.01f * (float) (i + 1); std::vector noise((size_t) cfg.max_action_dim * (size_t) cfg.n_suffix); - for (size_t i = 0; i < noise.size(); ++i) noise[i] = 0.001f * (float) ((i * 2654435761u) % 1000) - 0.5f; + for (size_t i = 0; i < noise.size(); ++i) + noise[i] = 0.001f * (float) ((i * 2654435761u) % 1000) - 0.5f; vla::Inputs in{}; in.images = views.data(); @@ -131,7 +157,11 @@ int main(int argc, char ** argv) { in.noise = noise.data(); for (int i = 0; i < warmup; ++i) { - if (vla::predict(m, in).empty()) { std::fprintf(stderr, "vla-bench: predict failed\n"); vla::model_free(m); return 1; } + if (vla::predict(m, in).empty()) { + std::fprintf(stderr, "vla-bench: predict failed\n"); + vla::model_free(m); + return 1; + } } std::vector ms; @@ -141,7 +171,11 @@ int main(int argc, char ** argv) { const auto t0 = std::chrono::steady_clock::now(); const std::vector out = vla::predict(m, in); const auto t1 = std::chrono::steady_clock::now(); - if (out.empty()) { std::fprintf(stderr, "vla-bench: predict failed at rep %d\n", i); vla::model_free(m); return 1; } + if (out.empty()) { + std::fprintf(stderr, "vla-bench: predict failed at rep %d\n", i); + vla::model_free(m); + return 1; + } ms.push_back(std::chrono::duration(t1 - t0).count()); vision_sum += vla::last_stats(m).ms_vision; } diff --git a/src/serving/vla-cli.cpp b/src/serving/vla-cli.cpp index c2fa374..9030e3c 100644 --- a/src/serving/vla-cli.cpp +++ b/src/serving/vla-cli.cpp @@ -51,13 +51,21 @@ bool parse_ints(const std::string & s, std::vector & out) { out.clear(); size_t i = 0; while (i < s.size()) { - while (i < s.size() && (s[i] == ',' || s[i] == ' ')) ++i; - if (i >= s.size()) break; + while (i < s.size() && (s[i] == ',' || s[i] == ' ')) + ++i; + if (i >= s.size()) + break; errno = 0; char * e = nullptr; long long x = std::strtoll(s.c_str() + i, &e, 10); - if (e == s.c_str() + i) { std::fprintf(stderr, "vla-cli: bad token near '%s'\n", s.c_str() + i); return false; } - if (errno == ERANGE || x < INT32_MIN || x > INT32_MAX) { std::fprintf(stderr, "vla-cli: token %lld out of int32 range\n", x); return false; } + if (e == s.c_str() + i) { + std::fprintf(stderr, "vla-cli: bad token near '%s'\n", s.c_str() + i); + return false; + } + if (errno == ERANGE || x < INT32_MIN || x > INT32_MAX) { + std::fprintf(stderr, "vla-cli: token %lld out of int32 range\n", x); + return false; + } out.push_back((int32_t) x); i = (size_t) (e - s.c_str()); } @@ -69,12 +77,20 @@ bool parse_floats(const std::string & s, std::vector & out) { out.clear(); size_t i = 0; while (i < s.size()) { - while (i < s.size() && (s[i] == ',' || s[i] == ' ')) ++i; - if (i >= s.size()) break; + while (i < s.size() && (s[i] == ',' || s[i] == ' ')) + ++i; + if (i >= s.size()) + break; char * e = nullptr; float x = std::strtof(s.c_str() + i, &e); - if (e == s.c_str() + i) { std::fprintf(stderr, "vla-cli: bad number near '%s'\n", s.c_str() + i); return false; } - if (!std::isfinite(x)) { std::fprintf(stderr, "vla-cli: non-finite value in --state\n"); return false; } + if (e == s.c_str() + i) { + std::fprintf(stderr, "vla-cli: bad number near '%s'\n", s.c_str() + i); + return false; + } + if (!std::isfinite(x)) { + std::fprintf(stderr, "vla-cli: non-finite value in --state\n"); + return false; + } out.push_back(x); i = (size_t) (e - s.c_str()); } @@ -113,12 +129,14 @@ const char * arch_slug(Arch a) { // The instruction reaches a shell command, so keep it to plain prose. bool text_ok(const std::string & s) { - if (s.empty() || s.size() > 512) return false; + if (s.empty() || s.size() > 512) + return false; for (const char c : s) { const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == ' ' || c == '.' || c == ',' || c == '-' || c == '_' || c == '\''; - if (!ok) return false; + if (!ok) + return false; } return true; } @@ -136,7 +154,12 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { return ""; } std::string esc; - for (const char c : text) { if (c == '\'') esc += "'\\''"; else esc += c; } + for (const char c : text) { + if (c == '\'') + esc += "'\\''"; + else + esc += c; + } // Env first so a packaged binary can point at its own copy of the script. const char * env = std::getenv("VLA_TOKENIZE_SCRIPT"); const std::string script = (env && *env) ? std::string(env) @@ -147,10 +170,14 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { " --text '" + esc + "'"; FILE * fp = popen(cmd.c_str(), "r"); - if (!fp) { std::fprintf(stderr, "vla-cli: cannot run %s\n", cmd.c_str()); return ""; } + if (!fp) { + std::fprintf(stderr, "vla-cli: cannot run %s\n", cmd.c_str()); + return ""; + } std::string out; char buf[4096]; - while (std::fgets(buf, sizeof(buf), fp)) out += buf; + while (std::fgets(buf, sizeof(buf), fp)) + out += buf; if (pclose(fp) != 0) { std::fprintf(stderr, "vla-cli: tokenizing failed. Install the client extras with\n" @@ -158,7 +185,8 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { " (VLA_PYTHON selects a different interpreter)\n"); return ""; } - while (!out.empty() && (out.back() == '\n' || out.back() == '\r')) out.pop_back(); + while (!out.empty() && (out.back() == '\n' || out.back() == '\r')) + out.pop_back(); return out; } @@ -187,10 +215,14 @@ int main(int argc, char ** argv) { for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto need = [&](const char * name) -> const char * { - if (i + 1 >= argc) { std::fprintf(stderr, "vla-cli: %s needs a value\n", name); std::exit(1); } + if (i + 1 >= argc) { + std::fprintf(stderr, "vla-cli: %s needs a value\n", name); + std::exit(1); + } return argv[++i]; }; - if (a == "--mmproj") mmproj = need("--mmproj"); + if (a == "--mmproj") + mmproj = need("--mmproj"); else if (a == "--ckpt") ckpt = need("--ckpt"); else if (a == "-hf") hf = need("-hf"); else if (a == "--image") image_paths.push_back(need("--image")); @@ -198,30 +230,55 @@ int main(int argc, char ** argv) { else if (a == "--text") text_s = need("--text"); else if (a == "--state") state_s = need("--state"); else if (a == "--pretty") pretty = true; - else if (a == "-h" || a == "--help") { usage(argv[0]); return 0; } - else { std::fprintf(stderr, "vla-cli: unknown argument %s\n", a.c_str()); usage(argv[0]); return 1; } + else if (a == "-h" || a == "--help") { + usage(argv[0]); + return 0; + } + else { + std::fprintf(stderr, "vla-cli: unknown argument %s\n", a.c_str()); + usage(argv[0]); + return 1; + } } if (!hf.empty()) { - if (!ckpt.empty()) { std::fprintf(stderr, "vla-cli: pass --ckpt or -hf, not both\n"); return 1; } + if (!ckpt.empty()) { + std::fprintf(stderr, "vla-cli: pass --ckpt or -hf, not both\n"); + return 1; + } ckpt = vla::hf_resolve(hf); - if (ckpt.empty()) return 1; + if (ckpt.empty()) + return 1; + } + if (ckpt.empty() || image_paths.empty() || (tokens_s.empty() && text_s.empty())) { + usage(argv[0]); + return 1; + } + if (!tokens_s.empty() && !text_s.empty()) { + std::fprintf(stderr, "vla-cli: pass --text or --tokens, not both\n"); + return 1; } - if (ckpt.empty() || image_paths.empty() || (tokens_s.empty() && text_s.empty())) { usage(argv[0]); return 1; } - if (!tokens_s.empty() && !text_s.empty()) { std::fprintf(stderr, "vla-cli: pass --text or --tokens, not both\n"); return 1; } if (!text_s.empty()) { tokens_s = tokenize_text(ckpt, text_s); - if (tokens_s.empty()) return 1; + if (tokens_s.empty()) + return 1; std::fprintf(stderr, "vla-cli: --text tokenized to %s\n", tokens_s.c_str()); } // Validate the cheap args before loading the model. std::vector lang; std::vector state; - if (!parse_ints(tokens_s, lang) || !parse_floats(state_s, state)) return 1; - if (lang.empty()) { std::fprintf(stderr, "vla-cli: --tokens parsed to nothing\n"); return 1; } + if (!parse_ints(tokens_s, lang) || !parse_floats(state_s, state)) + return 1; + if (lang.empty()) { + std::fprintf(stderr, "vla-cli: --tokens parsed to nothing\n"); + return 1; + } Model * m = model_load(mmproj, ckpt, ""); - if (!m) { std::fprintf(stderr, "vla-cli: model_load failed\n"); return 1; } + if (!m) { + std::fprintf(stderr, "vla-cli: model_load failed\n"); + return 1; + } const Config & cfg = model_config(m); if (!state.empty() && (int64_t) state.size() != cfg.max_state_dim) @@ -233,7 +290,10 @@ int main(int argc, char ** argv) { std::vector views(image_paths.size()); for (size_t v = 0; v < image_paths.size(); ++v) { int w = 0, h = 0; - if (!load_image(image_paths[v].c_str(), imgbuf[v], w, h)) { model_free(m); return 1; } + if (!load_image(image_paths[v].c_str(), imgbuf[v], w, h)) { + model_free(m); + return 1; + } views[v] = ImageView{ imgbuf[v].data(), w, h, PixelFormat::U8 }; } @@ -246,7 +306,11 @@ int main(int argc, char ** argv) { in.noise = nullptr; // predict() samples N(0,1) when omitted std::vector act = predict(m, in); - if (act.empty()) { std::fprintf(stderr, "vla-cli: predict failed\n"); model_free(m); return 2; } + if (act.empty()) { + std::fprintf(stderr, "vla-cli: predict failed\n"); + model_free(m); + return 2; + } const int64_t cols = cfg.max_action_dim > 0 ? cfg.max_action_dim : 1; if (pretty) { @@ -254,7 +318,8 @@ int main(int argc, char ** argv) { std::printf("%.6g%c", act[i], ((int64_t) (i + 1) % cols == 0) ? '\n' : ' '); } else { std::printf("action_len=%zu\n", act.size()); - for (float x : act) std::printf("%.9g\n", x); + for (float x : act) + std::printf("%.9g\n", x); } std::fflush(stdout); diff --git a/src/serving/vlm-server.cpp b/src/serving/vlm-server.cpp index 2f8f75f..05dff9e 100644 --- a/src/serving/vlm-server.cpp +++ b/src/serving/vlm-server.cpp @@ -37,7 +37,9 @@ namespace { std::atomic g_shutdown{false}; -void on_signal(int) { g_shutdown.store(true, std::memory_order_relaxed); } +void on_signal(int) { + g_shutdown.store(true, std::memory_order_relaxed); +} // Reject absurd image dimensions before any size arithmetic on untrusted input. constexpr unsigned kMaxImageDim = 8192; @@ -137,12 +139,15 @@ int main(int argc, char ** argv) { try { zmq::poll(poll, 1, std::chrono::milliseconds(200)); } catch (const zmq::error_t & e) { - if (e.num() == EINTR) continue; - if (e.num() == ETERM) break; + if (e.num() == EINTR) + continue; + if (e.num() == ETERM) + break; std::fprintf(stderr, "vlm-server: zmq error: %s\n", e.what()); continue; } - if (!(poll[0].revents & ZMQ_POLLIN)) continue; + if (!(poll[0].revents & ZMQ_POLLIN)) + continue; // maxmsgsize bounds each frame but not how many, so a peer could stream // sub-limit frames until memory runs out. @@ -157,7 +162,10 @@ int main(int argc, char ** argv) { zmq::message_t part; try { auto rr = sock.recv(part, zmq::recv_flags::none); - if (!rr) { recv_ok = false; break; } + if (!rr) { + recv_ok = false; + break; + } } catch (const zmq::error_t & e) { if (e.num() != EINTR) std::fprintf(stderr, "vlm-server: zmq recv error: %s\n", e.what()); @@ -181,7 +189,8 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "vlm-server: oversized ROUTER envelope; request dropped\n"); continue; } - if (!recv_ok || !have_payload || env.empty()) continue; + if (!recv_ok || !have_payload || env.empty()) + continue; auto send_reply = [&](const std::string & body) { try { @@ -218,7 +227,8 @@ int main(int argc, char ** argv) { continue; } size_t text_bytes = 0; - for (const auto & m : req.messages()) text_bytes += m.content().size(); + for (const auto & m : req.messages()) + text_bytes += m.content().size(); if (text_bytes > kMaxTextBytes) { send_reply(make_error_stream(rid, "message text too large (max 4 MiB)")); continue; @@ -281,7 +291,8 @@ int main(int argc, char ** argv) { } images.push_back(std::move(out)); } - if (!decode_ok) continue; + if (!decode_ok) + continue; std::vector messages; messages.reserve(req.messages_size()); @@ -330,7 +341,8 @@ int main(int argc, char ** argv) { resp->set_latency_ms_total(ms_total); resp->set_latency_ms_prefill(r.ms_prefill); resp->set_latency_ms_decode(r.ms_decode); - if (r.finish_reason == "error") resp->set_error(r.error); + if (r.finish_reason == "error") + resp->set_error(r.error); send_reply(sm.SerializeAsString()); ++served; diff --git a/src/vla_c_api.cpp b/src/vla_c_api.cpp index 74c10bc..3c38fdf 100644 --- a/src/vla_c_api.cpp +++ b/src/vla_c_api.cpp @@ -41,16 +41,20 @@ struct vla_model : vla_model_impl {}; extern "C" { -int32_t vla_abi_version(void) { return VLA_ABI_VERSION; } +int32_t vla_abi_version(void) { + return VLA_ABI_VERSION; +} vla_model * vla_model_load(const char * mmproj_path, const char * ckpt_path, const char * config_path) { - if (!ckpt_path) return nullptr; + if (!ckpt_path) + return nullptr; try { vla::Model * m = vla::model_load(mmproj_path ? mmproj_path : "", ckpt_path, config_path ? config_path : ""); - if (!m) return nullptr; + if (!m) + return nullptr; // Owns the engine until the handle exists, so a throwing new does not // strand the whole model. std::unique_ptr guard(m, vla::model_free); @@ -63,13 +67,15 @@ vla_model * vla_model_load(const char * mmproj_path, const char * ckpt_path, } void vla_model_free(vla_model * h) { - if (!h) return; + if (!h) + return; vla::model_free(h->m); delete h; } int32_t vla_model_config(const vla_model * h, vla_config * out) { - if (!h || !h->m || !out) return VLA_ERR_ARG; + if (!h || !h->m || !out) + return VLA_ERR_ARG; try { const vla::Config & c = vla::model_config(h->m); *out = vla_config{}; @@ -111,12 +117,16 @@ int32_t vla_model_config(const vla_model * h, vla_config * out) { int32_t vla_predict(vla_model * h, const vla_inputs * in, float ** out_actions, int64_t * out_n) { - if (!h || !h->m || !in || !out_actions || !out_n) return VLA_ERR_ARG; + if (!h || !h->m || !in || !out_actions || !out_n) + return VLA_ERR_ARG; *out_actions = nullptr; *out_n = 0; - if (in->n_images < 0 || in->n_lang < 0 || in->n_img_views < 0) return VLA_ERR_ARG; - if (in->n_images > 0 && !in->images) return VLA_ERR_ARG; - if (in->n_lang > 0 && !in->lang_tokens) return VLA_ERR_ARG; + if (in->n_images < 0 || in->n_lang < 0 || in->n_img_views < 0) + return VLA_ERR_ARG; + if (in->n_images > 0 && !in->images) + return VLA_ERR_ARG; + if (in->n_lang > 0 && !in->lang_tokens) + return VLA_ERR_ARG; try { std::vector views((size_t) (in->n_images > 0 ? in->n_images : 0)); @@ -143,11 +153,13 @@ int32_t vla_predict(vla_model * h, const vla_inputs * in, : vla::TimingDetail::NONE; const std::vector act = vla::predict(h->m, ci); - if (act.empty()) return VLA_ERR_PREDICT; + if (act.empty()) + return VLA_ERR_PREDICT; // malloc pairs with vla_free_actions, which callers may replace. float * buf = (float *) std::malloc(act.size() * sizeof(float)); - if (!buf) return VLA_ERR_EXCEPTION; + if (!buf) + return VLA_ERR_EXCEPTION; std::memcpy(buf, act.data(), act.size() * sizeof(float)); *out_actions = buf; *out_n = (int64_t) act.size(); @@ -157,10 +169,13 @@ int32_t vla_predict(vla_model * h, const vla_inputs * in, } } -void vla_free_actions(float * actions) { std::free(actions); } +void vla_free_actions(float * actions) { + std::free(actions); +} int32_t vla_last_stats(const vla_model * h, vla_stats * out) { - if (!h || !h->m || !out) return VLA_ERR_ARG; + if (!h || !h->m || !out) + return VLA_ERR_ARG; try { const vla::Stats & s = vla::last_stats(h->m); out->ms_total = s.ms_total; diff --git a/src/vlm/engine.cpp b/src/vlm/engine.cpp index 8f5e81f..57cfd34 100644 --- a/src/vlm/engine.cpp +++ b/src/vlm/engine.cpp @@ -61,7 +61,9 @@ struct Engine::Impl { Engine::Engine() : impl_(std::make_unique()) {} Engine::~Engine() = default; -bool Engine::loaded() const { return impl_ && impl_->lctx != nullptr; } +bool Engine::loaded() const { + return impl_ && impl_->lctx != nullptr; +} bool Engine::load(const LoadParams & lp) { ensure_global_init(); @@ -179,7 +181,10 @@ ChatResult Engine::chat(const std::vector & messages, int img_msg_idx = -1; for (int i = (int) messages.size() - 1; i >= 0; --i) { - if (messages[i].role == "user") { img_msg_idx = i; break; } + if (messages[i].role == "user") { + img_msg_idx = i; + break; + } } if (img_msg_idx < 0) { img_msg_idx = (int) messages.size() - 1; @@ -198,7 +203,8 @@ ChatResult Engine::chat(const std::vector & messages, if (msg.content.find(marker) == std::string::npos) { std::string prefix; - for (size_t k = 0; k < images.size(); ++k) prefix += marker; + for (size_t k = 0; k < images.size(); ++k) + prefix += marker; msg.content = prefix + msg.content; } for (const auto & im : images) { @@ -224,7 +230,8 @@ ChatResult Engine::chat(const std::vector & messages, text.parse_special = true; std::vector bmp_ptrs(bmps.size()); - for (size_t k = 0; k < bmps.size(); ++k) bmp_ptrs[k] = bmps[k].ptr.get(); + for (size_t k = 0; k < bmps.size(); ++k) + bmp_ptrs[k] = bmps[k].ptr.get(); mtmd::input_chunks chunks(mtmd_input_chunks_init()); if (mtmd_tokenize(impl_->vision.get(), chunks.ptr.get(), &text, From 9678dc4be8d935941127f6ab34ec45ecea6cea03 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 23:21:11 +0700 Subject: [PATCH 16/21] tighten spacing around arithmetic operators across src --- scripts/tighten_ops.py | 70 ++++++ src/act_dtype.h | 2 +- src/arch.h | 6 +- src/cuda/vla_cuda_bf16.cu | 154 ++++++------ src/env_flag.h | 2 +- src/gguf_reader.h | 14 +- src/kernels/bitvla/bitnet_kernels.h | 168 +++++++------- src/kernels/bitvla/bitvla_fp32head_cuda.cu | 54 ++--- src/kernels/bitvla/bitvla_fp32head_cuda.h | 6 +- src/kernels/bitvla/bitvla_lm_cuda.cu | 144 ++++++------ src/kernels/bitvla/bitvla_lm_cuda.h | 8 +- src/kernels/bitvla/bitvla_vit_cuda.cu | 24 +- src/kernels/bitvla/bitvla_vit_cuda.h | 2 +- src/model.cpp | 2 +- src/model.h | 2 +- src/models/bitvla.cpp | 258 ++++++++++----------- src/models/dit_common.h | 24 +- src/models/evo1.cpp | 78 +++---- src/models/gr00tn1d7.cpp | 130 +++++------ src/models/openvla_oft.cpp | 18 +- src/models/pi0.cpp | 70 +++--- src/models/pi05.cpp | 64 ++--- src/models/smolvla.cpp | 242 +++++++++---------- src/models/vla_adapter.cpp | 12 +- src/models/vla_jepa.cpp | 92 ++++---- src/modules/preprocess.h | 18 +- src/modules/qwen3vl_vit.h | 54 ++--- src/serving/hf_fetch.h | 4 +- src/serving/server.cpp | 26 +-- src/serving/vla-bench.cpp | 24 +- src/serving/vla-cli.cpp | 22 +- src/serving/vlm-server.cpp | 16 +- src/vla_c_api.cpp | 4 +- src/vlm/engine.cpp | 14 +- src/vlm/engine.h | 2 +- 35 files changed, 950 insertions(+), 880 deletions(-) create mode 100644 scripts/tighten_ops.py diff --git a/scripts/tighten_ops.py b/scripts/tighten_ops.py new file mode 100644 index 0000000..2cb3cb7 --- /dev/null +++ b/scripts/tighten_ops.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics - Apache-2.0 +# +# Removes the spaces around binary operators, per the house style. Whitespace +# only, and deliberately conservative: it edits an operator only when both +# sides are unambiguously values. +# +# Left untouched on purpose: +# * & unless one side is a literal, a ')' / ']', or a '.'/'->' member, +# because `ggml_tensor * t` is a declaration, not a product +# < > template brackets and includes are indistinguishable here +# = `a = b` reads better spaced outside a for-header +# any operator whose tightening would merge two tokens (`a - -b`) +# +# Strings, character literals, comments and preprocessor lines are copied +# through untouched. + +import re +import sys + +sys.path.insert(0, __file__.rsplit('/', 1)[0]) +from restyle import spans_to_skip + +VALUE_END = re.compile(r'(?:[A-Za-z_]\w*|\d[\w.]*|[)\]])$') +VALUE_START = re.compile(r'^(?:[A-Za-z_]\w*|\d|[(\[])') +MEMBER = re.compile(r'(?:\.\s*[A-Za-z_]\w*|->\s*[A-Za-z_]\w*|\)|\]|\d)$') + +OPS = ['+', '-', '/', '%'] + +def tighten(line): + if line.lstrip().startswith('#'): + return line + + skip = spans_to_skip(line) + live = lambda i: not any(a <= i < b for a, b in skip) + + out = line + for _ in range(40): + changed = False + for m in re.finditer(r'(?<=\S) (' + '|'.join(re.escape(o) for o in OPS + ['*']) + r') (?=\S)', out): + i, op = m.start(), m.group(1) + if not live(i): + continue + left, right = out[:i], out[i + 3:] + if not VALUE_END.search(left) or not VALUE_START.match(right): + continue + # `a * b` is only a product when one side is plainly a value. + if op == '*' and not (MEMBER.search(left) or right[0].isdigit()): + continue + # never let the operator glue onto a neighbour + if left[-1] == op or right[0] in '=&|<>+-' or right[0] == op: + continue + out = left + op + right + skip = spans_to_skip(out) + changed = True + break + if not changed: + break + return out + +if __name__ == '__main__': + total = 0 + for path in sys.argv[1:]: + src = open(path).read() + dst = '\n'.join(tighten(l) for l in src.split('\n')) + n = sum(1 for a, b in zip(src.split('\n'), dst.split('\n')) if a != b) + if n: + open(path, 'w').write(dst) + total += n + print(f'tightened {total} lines') diff --git a/src/act_dtype.h b/src/act_dtype.h index 7463676..d74cd84 100644 --- a/src/act_dtype.h +++ b/src/act_dtype.h @@ -62,7 +62,7 @@ inline ggml_tensor * as_type(ggml_context * C, ggml_tensor * t, ggml_type ty) { inline ggml_tensor * mul_mat_t(ggml_context * C, ggml_tensor * a, ggml_tensor * b, ggml_type type) { // ggml_can_mul_mat is internal to ggml; this is the same condition. GGML_ASSERT(a->ne[0] == b->ne[0]); - GGML_ASSERT(b->ne[2] % a->ne[2] == 0 && b->ne[3] % a->ne[3] == 0); + GGML_ASSERT(b->ne[2]%a->ne[2] == 0 && b->ne[3]%a->ne[3] == 0); GGML_ASSERT(!ggml_is_transposed(a)); const int64_t ne[4] = { a->ne[1], b->ne[1], b->ne[2], b->ne[3] }; diff --git a/src/arch.h b/src/arch.h index cdf1f63..45db734 100644 --- a/src/arch.h +++ b/src/arch.h @@ -95,9 +95,9 @@ class ModelArchBase { /** * @brief Run a full forward pass and return one chunk of normalised actions. - * @param in Vision + language + state inputs (see @ref Inputs). + * @param in Vision+language+state inputs (see @ref Inputs). * @return Flattened action chunk of length - * @c cfg.num_steps * cfg.real_action_dim. + * @c cfg.num_steps*cfg.real_action_dim. */ virtual std::vector predict(const Inputs& in) = 0; }; @@ -105,7 +105,7 @@ class ModelArchBase { /** * @brief Build a SmolVLA model from its mmproj and checkpoint GGUFs. * @param mmproj_path Path to the vision-tower GGUF. - * @param ckpt_path Path to the LM + action-expert GGUF. + * @param ckpt_path Path to the LM+action-expert GGUF. * @param config_path Optional JSON override; pass empty to use bundled config. * @return Owning pointer to the constructed model. */ diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index d9dafe0..72cf2e3 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -76,7 +76,7 @@ inline __device__ __nv_bfloat16 f2bf(const float v) { enum class BinOp { Add, Mul }; inline __device__ float apply_bin(BinOp op, float a, float b) { - return op == BinOp::Add ? a + b : a * b; + return op == BinOp::Add ? a+b : a * b; } // Broadcast index along one dimension. ggml's rule is a modulo, but the only @@ -88,7 +88,7 @@ inline __device__ int64_t bcast_idx(int64_t i, int64_t ne_src, int64_t ne_dst) { return i; if (ne_src == 1) return 0; - return i % ne_src; + return i%ne_src; } // Rows are addressed through blockIdx.y/z rather than recovered from a flat @@ -98,8 +98,8 @@ inline __device__ int64_t bcast_idx(int64_t i, int64_t ne_src, int64_t ne_dst) { // evo1 call that difference was larger than everything BF16 activations saved. template __global__ void k_bin_bcast_bf16_rows( - const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, - __nv_bfloat16 * __restrict__ dst, + const __nv_bfloat16*__restrict__ src0, const S1*__restrict__ src1, + __nv_bfloat16*__restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, @@ -107,18 +107,18 @@ __global__ void k_bin_bcast_bf16_rows( const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { const int64_t i1 = blockIdx.y; const int64_t i23 = blockIdx.z; - const int64_t i2 = i23 % ne2; // once per block, not per element - const int64_t i3 = i23 / ne2; + const int64_t i2 = i23%ne2; // once per block, not per element + const int64_t i3 = i23/ne2; const int64_t j1 = bcast_idx(i1, ne11, ne1); const int64_t j2 = bcast_idx(i2, ne12, ne2); const int64_t j3 = bcast_idx(i3, ne13, ne3); - const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; - const S1 * __restrict__ r1 = src1 + j1*s11 + j2*s12 + j3*s13; + const __nv_bfloat16*__restrict__ r0 = src0+i1*s01+i2*s02+i3*s03; + const S1 * __restrict__ r1 = src1+j1*s11+j2*s12+j3*s13; __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; - for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i0 < ne0; + for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; i0 < ne0; i0 += (int64_t) gridDim.x*blockDim.x) { const float a = bf2f(r0[i0*s00]); const float b = (float) r1[bcast_idx(i0, ne10, ne0)*s10]; @@ -135,8 +135,8 @@ __global__ void k_bin_bcast_bf16_rows( // their mean/median split (6.3 vs 2.2 us) says the big tensors carry the total. template __global__ void k_bin_bcast_bf16_vec8( - const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, - __nv_bfloat16 * __restrict__ dst, + const __nv_bfloat16*__restrict__ src0, const S1*__restrict__ src1, + __nv_bfloat16*__restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne11, const int64_t ne12, const int64_t ne13, @@ -144,61 +144,61 @@ __global__ void k_bin_bcast_bf16_vec8( const int64_t d1, const int64_t d2, const int64_t d3) { const int64_t i1 = blockIdx.y; const int64_t i23 = blockIdx.z; - const int64_t i2 = i23 % ne2; - const int64_t i3 = i23 / ne2; + const int64_t i2 = i23%ne2; + const int64_t i3 = i23/ne2; - const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; - const S1 * __restrict__ r1 = src1 + bcast_idx(i1, ne11, ne1)*s11 + + const __nv_bfloat16*__restrict__ r0 = src0+i1*s01+i2*s02+i3*s03; + const S1 * __restrict__ r1 = src1+bcast_idx(i1, ne11, ne1)*s11 + bcast_idx(i2, ne12, ne2)*s12 + bcast_idx(i3, ne13, ne3)*s13; __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; - const int64_t nvec = ne0 / 8; - for (int64_t v = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; v < nvec; + const int64_t nvec = ne0/8; + for (int64_t v = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; v < nvec; v += (int64_t) gridDim.x*blockDim.x) { - const int64_t i0 = v * 8; + const int64_t i0 = v*8; - uint4 a = *reinterpret_cast(r0 + i0); - __nv_bfloat16 * av = reinterpret_cast<__nv_bfloat16 *>(&a); + uint4 a = *reinterpret_cast(r0+i0); + __nv_bfloat16*av = reinterpret_cast<__nv_bfloat16 *>(&a); #pragma unroll for (int k = 0; k < 8; ++k) { - const float b = (float) r1[i0 + k]; + const float b = (float) r1[i0+k]; av[k] = f2bf(apply_bin(op, bf2f(av[k]), b)); } - *reinterpret_cast(rd + i0) = a; + *reinterpret_cast(rd+i0) = a; } } // Fallback for shapes the row grid cannot address (gridDim.y/z cap at 65535). template __global__ void k_bin_bcast_bf16_flat( - const __nv_bfloat16 * __restrict__ src0, const S1 * __restrict__ src1, - __nv_bfloat16 * __restrict__ dst, + const __nv_bfloat16*__restrict__ src0, const S1*__restrict__ src1, + __nv_bfloat16*__restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, const int64_t s10, const int64_t s11, const int64_t s12, const int64_t s13, const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { const int64_t total = ne0*ne1*ne2*ne3; - for (int64_t idx = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; idx < total; + for (int64_t idx = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; idx < total; idx += (int64_t) gridDim.x*blockDim.x) { const int64_t i0 = idx % ne0; - const int64_t i1 = (idx / ne0) % ne1; - const int64_t i2 = (idx / (ne0*ne1)) % ne2; - const int64_t i3 = idx / (ne0*ne1*ne2); + const int64_t i1 = (idx/ne0) % ne1; + const int64_t i2 = (idx/(ne0*ne1)) % ne2; + const int64_t i3 = idx/(ne0*ne1*ne2); - const float a = bf2f(src0[i0*s00 + i1*s01 + i2*s02 + i3*s03]); - const float b = (float) src1[(i0 % ne10)*s10 + (i1 % ne11)*s11 + - (i2 % ne12)*s12 + (i3 % ne13)*s13]; + const float a = bf2f(src0[i0*s00+i1*s01+i2*s02+i3*s03]); + const float b = (float) src1[(i0%ne10)*s10+(i1%ne11)*s11 + + (i2%ne12)*s12+(i3%ne13)*s13]; - dst[i0*d0 + i1*d1 + i2*d2 + i3*d3] = f2bf(apply_bin(op, a, b)); + dst[i0*d0+i1*d1+i2*d2+i3*d3] = f2bf(apply_bin(op, a, b)); } } // element strides (ggml stores byte strides) inline int64_t es(const ggml_tensor * t, int i) { - return t->nb[i] / ggml_type_size(t->type); + return t->nb[i]/ggml_type_size(t->type); } // Launch shape for the row-addressed kernels; ok=false means fall back to flat. @@ -227,7 +227,7 @@ inline RowGrid row_grid(int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { unsigned bx = 32; while (bx < (unsigned) BLOCK && (int64_t) bx < ne0) bx *= 2; - int64_t gx = (ne0 + bx - 1) / bx; + int64_t gx = (ne0+bx-1)/bx; if (gx > 65535) gx = 65535; if (gx < 1) @@ -263,17 +263,17 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { // Requires 16 B alignment for the uint4 accesses, which the ggml allocator // gives at tensor start but not necessarily at a strided row offset. const bool vec8_shape = - g.ok && dst->ne[0] % 8 == 0 && + g.ok && dst->ne[0]%8 == 0 && es(src0, 0) == 1 && es(dst, 0) == 1 && es(src1, 0) == 1 && src1->ne[0] == dst->ne[0] && - es(src0, 1) % 8 == 0 && es(dst, 1) % 8 == 0 && es(src1, 1) % 8 == 0 && - ((uintptr_t) src0->data % 16) == 0 && ((uintptr_t) dst->data % 16) == 0; + es(src0, 1)%8 == 0 && es(dst, 1)%8 == 0 && es(src1, 1)%8 == 0 && + ((uintptr_t) src0->data%16) == 0 && ((uintptr_t) dst->data%16) == 0; if (vec8_shape) { - const int64_t nvec = dst->ne[0] / 8; + const int64_t nvec = dst->ne[0]/8; unsigned bx = 32; while (bx < (unsigned) BLOCK && (int64_t) bx < nvec) bx *= 2; - int64_t gx = (nvec + bx - 1) / bx; + int64_t gx = (nvec+bx-1)/bx; if (gx > 65535) gx = 65535; if (gx < 1) @@ -313,7 +313,7 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { es(src1,0), es(src1,1), es(src1,2), es(src1,3), \ es(dst,0), es(dst,1), es(dst,2), es(dst,3)); \ } else { \ - const int64_t blocks = (ggml_nelements(dst) + BLOCK - 1) / BLOCK; \ + const int64_t blocks = (ggml_nelements(dst)+BLOCK-1)/BLOCK; \ const int flat = (int) (blocks < 65535 ? blocks : 65535); \ k_bin_bcast_bf16_flat<<>>( \ (const __nv_bfloat16 *) src0->data, (const TYPE *) src1->data, \ @@ -353,12 +353,12 @@ bool bin_bcast(ggml_tensor * dst, cudaStream_t stream) { constexpr int MAX_FUSE = 8; template -struct SrcPtrs { const S1 * p[MAX_FUSE]; }; +struct SrcPtrs { const S1*p[MAX_FUSE]; }; template __global__ void k_fused_bin_bcast_bf16( - const __nv_bfloat16 * __restrict__ src0, const SrcPtrs srcs, const int n_fuse, - __nv_bfloat16 * __restrict__ dst, + const __nv_bfloat16*__restrict__ src0, const SrcPtrs srcs, const int n_fuse, + __nv_bfloat16*__restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, @@ -366,19 +366,19 @@ __global__ void k_fused_bin_bcast_bf16( const int64_t d0, const int64_t d1, const int64_t d2, const int64_t d3) { const int64_t i1 = blockIdx.y; const int64_t i23 = blockIdx.z; - const int64_t i2 = i23 % ne2; - const int64_t i3 = i23 / ne2; + const int64_t i2 = i23%ne2; + const int64_t i3 = i23/ne2; const int64_t row1 = bcast_idx(i1, ne11, ne1)*s11 + bcast_idx(i2, ne12, ne2)*s12 + bcast_idx(i3, ne13, ne3)*s13; - const __nv_bfloat16 * __restrict__ r0 = src0 + i1*s01 + i2*s02 + i3*s03; + const __nv_bfloat16*__restrict__ r0 = src0+i1*s01+i2*s02+i3*s03; __nv_bfloat16 * __restrict__ rd = dst + i1*d1 + i2*d2 + i3*d3; - for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i0 < ne0; + for (int64_t i0 = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; i0 < ne0; i0 += (int64_t) gridDim.x*blockDim.x) { - const int64_t j = row1 + bcast_idx(i0, ne10, ne0)*s10; + const int64_t j = row1+bcast_idx(i0, ne10, ne0)*s10; float acc = bf2f(r0[i0*s00]); for (int k = 0; k < n_fuse; ++k) { @@ -413,7 +413,7 @@ bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { return false; for (int k = 1; k < n_fuse; ++k) { - const ggml_tensor * s = dst->src[k + 1]; + const ggml_tensor * s = dst->src[k+1]; if (!s || s->type != src1->type) return false; if (!ggml_are_same_shape(s, src1)) @@ -434,7 +434,7 @@ bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { #define VLA_LAUNCH_FUSED(TYPE) \ do { \ SrcPtrs srcs{}; \ - for (int k = 0; k < n_fuse; ++k) srcs.p[k] = (const TYPE *) dst->src[k + 1]->data; \ + for (int k = 0; k < n_fuse; ++k) srcs.p[k] = (const TYPE *) dst->src[k+1]->data; \ k_fused_bin_bcast_bf16<<>>( \ (const __nv_bfloat16 *) src0->data, srcs, n_fuse, \ (__nv_bfloat16 *) dst->data, \ @@ -463,20 +463,20 @@ enum class UnOp { Silu, Relu, Gelu, GeluErf }; template inline __device__ float apply_unary(const float x) { if (op == UnOp::Silu) - return x / (1.0f + expf(-x)); + return x/(1.0f+expf(-x)); if (op == UnOp::Relu) return x > 0.0f ? x : 0.0f; if (op == UnOp::GeluErf) - return 0.5f*x*(1.0f + erff(x*0.70710678118654752440f)); + return 0.5f*x*(1.0f+erff(x*0.70710678118654752440f)); // tanh approximation, matching ggml's GGML_UNARY_OP_GELU const float c = 0.79788456080286535588f; // sqrt(2/pi) - return 0.5f*x*(1.0f + tanhf(c*(x + 0.044715f*x*x*x))); + return 0.5f*x*(1.0f+tanhf(c*(x+0.044715f*x*x*x))); } template -__global__ void k_unary_bf16(const __nv_bfloat16 * __restrict__ x, - __nv_bfloat16 * __restrict__ dst, const int64_t n) { - for (int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i < n; +__global__ void k_unary_bf16(const __nv_bfloat16*__restrict__ x, + __nv_bfloat16*__restrict__ dst, const int64_t n) { + for (int64_t i = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; i < n; i += (int64_t) gridDim.x*blockDim.x) { dst[i] = f2bf(apply_unary(bf2f(x[i]))); } @@ -492,7 +492,7 @@ bool unary(ggml_tensor * dst, cudaStream_t stream) { return false; const int64_t n = ggml_nelements(dst); - const int64_t blocks = (n + BLOCK - 1) / BLOCK; + const int64_t blocks = (n+BLOCK-1)/BLOCK; const int grid = (int) (blocks < 65535 ? blocks : 65535); k_unary_bf16<<>>( (const __nv_bfloat16 *) src0->data, (__nv_bfloat16 *) dst->data, n); @@ -503,11 +503,11 @@ bool unary(ggml_tensor * dst, cudaStream_t stream) { // scale: dst = x*scale + bias // --------------------------------------------------------------------------- -__global__ void k_scale_bf16(const __nv_bfloat16 * __restrict__ x, __nv_bfloat16 * __restrict__ dst, +__global__ void k_scale_bf16(const __nv_bfloat16*__restrict__ x, __nv_bfloat16*__restrict__ dst, const float scale, const float bias, const int64_t n) { - for (int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i < n; + for (int64_t i = (int64_t) blockIdx.x*blockDim.x+threadIdx.x; i < n; i += (int64_t) gridDim.x*blockDim.x) { - dst[i] = f2bf(scale*bf2f(x[i]) + bias); + dst[i] = f2bf(scale*bf2f(x[i])+bias); } } @@ -519,11 +519,11 @@ bool scale(ggml_tensor * dst, cudaStream_t stream) { return false; float s = 1.0f, b = 0.0f; - memcpy(&s, (const float *) dst->op_params + 0, sizeof(float)); - memcpy(&b, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&s, (const float *) dst->op_params+0, sizeof(float)); + memcpy(&b, (const float *) dst->op_params+1, sizeof(float)); const int64_t n = ggml_nelements(dst); - const int64_t blocks = (n + BLOCK - 1) / BLOCK; + const int64_t blocks = (n+BLOCK-1)/BLOCK; const int grid = (int) (blocks < 65535 ? blocks : 65535); k_scale_bf16<<>>( (const __nv_bfloat16 *) src0->data, (__nv_bfloat16 *) dst->data, s, b, n); @@ -538,22 +538,22 @@ __device__ inline float block_sum(float v, float * shared) { const int tid = threadIdx.x; shared[tid] = v; __syncthreads(); - for (int s = blockDim.x / 2; s > 0; s >>= 1) { + for (int s = blockDim.x/2; s > 0; s >>= 1) { if (tid < s) - shared[tid] += shared[tid + s]; + shared[tid] += shared[tid+s]; __syncthreads(); } return shared[0]; } template -__global__ void k_norm_bf16(const __nv_bfloat16 * __restrict__ x, __nv_bfloat16 * __restrict__ dst, +__global__ void k_norm_bf16(const __nv_bfloat16*__restrict__ x, __nv_bfloat16*__restrict__ dst, const int64_t ncols, const int64_t sx1, const int64_t sd1, const float eps) { __shared__ float shared[BLOCK]; const int64_t row = blockIdx.x; - const __nv_bfloat16 * xr = x + row*sx1; - __nv_bfloat16 * dr = dst + row*sd1; + const __nv_bfloat16*xr = x + row*sx1; + __nv_bfloat16 * dr = dst+row*sd1; float sum = 0.0f, sumsq = 0.0f; for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) { @@ -564,17 +564,17 @@ __global__ void k_norm_bf16(const __nv_bfloat16 * __restrict__ x, __nv_bfloat16 } if (rms) { - const float ms = block_sum(sumsq, shared) / (float) ncols; - const float inv = rsqrtf(ms + eps); + const float ms = block_sum(sumsq, shared)/(float) ncols; + const float inv = rsqrtf(ms+eps); for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) dr[c] = f2bf(bf2f(xr[c])*inv); } else { - const float mean = block_sum(sum, shared) / (float) ncols; + const float mean = block_sum(sum, shared)/(float) ncols; __syncthreads(); - const float meansq = block_sum(sumsq, shared) / (float) ncols; - const float inv = rsqrtf(meansq - mean*mean + eps); + const float meansq = block_sum(sumsq, shared)/(float) ncols; + const float inv = rsqrtf(meansq-mean*mean+eps); for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) - dr[c] = f2bf((bf2f(xr[c]) - mean)*inv); + dr[c] = f2bf((bf2f(xr[c])-mean)*inv); } } @@ -595,7 +595,7 @@ bool norm(ggml_tensor * dst, cudaStream_t stream) { memcpy(&eps, dst->op_params, sizeof(float)); const int64_t ncols = src0->ne[0]; - const int64_t nrows = ggml_nelements(src0) / ncols; + const int64_t nrows = ggml_nelements(src0)/ncols; if (nrows > 2147483647) return false; @@ -648,8 +648,8 @@ bool mul_mat(ggml_tensor * dst, cudaStream_t stream) { const int64_t ne12 = src1->ne[2], ne13 = src1->ne[3]; const int64_t ne0 = dst->ne[0], ne1 = dst->ne[1]; - const __nv_bfloat16 * a = (const __nv_bfloat16 *) src0->data; - const __nv_bfloat16 * b = (const __nv_bfloat16 *) src1->data; + const __nv_bfloat16*a = (const __nv_bfloat16 *) src0->data; + const __nv_bfloat16*b = (const __nv_bfloat16 *) src1->data; __nv_bfloat16 * c = (__nv_bfloat16 *) dst->data; const float alpha = 1.0f, beta = 0.0f; diff --git a/src/env_flag.h b/src/env_flag.h index b2a0b41..877c7a1 100644 --- a/src/env_flag.h +++ b/src/env_flag.h @@ -48,7 +48,7 @@ inline bool env_flag(const char * name, bool def = false) { char buf[8] = {}; size_t n = 0; - for (; n < sizeof(buf) - 1 && v[n]; ++n) { + for (; n < sizeof(buf)-1 && v[n]; ++n) { buf[n] = (char) std::tolower((unsigned char) v[n]); } if (v[n]) return true; // longer than any false word, so it is one diff --git a/src/gguf_reader.h b/src/gguf_reader.h index 23f8f26..82bf212 100644 --- a/src/gguf_reader.h +++ b/src/gguf_reader.h @@ -122,7 +122,7 @@ struct gguf_reader { std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch, name); return false; } - const size_t off = data_off + gguf_get_tensor_offset(gctx, id); + const size_t off = data_off+gguf_get_tensor_offset(gctx, id); const size_t nb = gguf_get_tensor_size(gctx, id); if (nb != cap) { std::fprintf(stderr, "vla(%s): tensor %s is %zu bytes, caller expects %zu\n", @@ -139,8 +139,8 @@ struct gguf_reader { if (!t) { std::fprintf(stderr, "vla(%s): missing tensor %s\n", arch, name); return {}; } const int64_t n = ggml_nelements(t); std::vector out(n); - if (t->type == GGML_TYPE_F32) { if (!read_raw(name, out.data(), out.size() * sizeof(float))) return {}; } - else if (t->type == GGML_TYPE_BF16) { std::vector tmp(n); if (!read_raw(name, tmp.data(), tmp.size() * sizeof(ggml_bf16_t))) return {}; ggml_bf16_to_fp32_row(tmp.data(), out.data(), n); } + if (t->type == GGML_TYPE_F32) { if (!read_raw(name, out.data(), out.size()*sizeof(float))) return {}; } + else if (t->type == GGML_TYPE_BF16) { std::vector tmp(n); if (!read_raw(name, tmp.data(), tmp.size()*sizeof(ggml_bf16_t))) return {}; ggml_bf16_to_fp32_row(tmp.data(), out.data(), n); } else { std::fprintf(stderr, "vla(%s): tensor %s unsupported type %d\n", arch, name, (int) t->type); return {}; } return out; } @@ -192,7 +192,7 @@ struct gguf_reader { } const int64_t rows = t->ne[1]; const int64_t id = gguf_find_tensor(gctx, name); - const size_t base = data_off + gguf_get_tensor_offset(gctx, id); + const size_t base = data_off+gguf_get_tensor_offset(gctx, id); const size_t elsz = (t->type == GGML_TYPE_F32) ? 4u : 2u; const size_t rb = (size_t) cols * elsz; std::vector row(rb); @@ -202,14 +202,14 @@ struct gguf_reader { std::fprintf(stderr, "vla(%s): row %d out of range for %s\n", arch, r, name); return false; } - if (fseeko(fp, (off_t) (base + (size_t) r * rb), SEEK_SET) != 0) + if (fseeko(fp, (off_t) (base+(size_t) r * rb), SEEK_SET) != 0) return false; if (std::fread(row.data(), 1, rb, fp) != rb) return false; if (elsz == 4) - std::memcpy(dst + k * cols, row.data(), rb); + std::memcpy(dst+k * cols, row.data(), rb); else - ggml_bf16_to_fp32_row(reinterpret_cast(row.data()), dst + k * cols, cols); + ggml_bf16_to_fp32_row(reinterpret_cast(row.data()), dst+k * cols, cols); } return true; } diff --git a/src/kernels/bitvla/bitnet_kernels.h b/src/kernels/bitvla/bitnet_kernels.h index 6420a81..7d758eb 100644 --- a/src/kernels/bitvla/bitnet_kernels.h +++ b/src/kernels/bitvla/bitnet_kernels.h @@ -24,8 +24,8 @@ * fragment multiply per tile. * * Two GEMM entry points are provided: - * * @ref ladder_int8xint2_kernel - single-row (M=1) decode kernel - * used for next-token / single-query inference. + * * @ref ladder_int8xint2_kernel-single-row (M=1) decode kernel + * used for next-token/single-query inference. * * @ref ladder_int8xint2_kernel_m + @ref launch_ladder_int8xint2_m * - multi-row (M>1) variant for prefill and ViT batches. * @@ -80,11 +80,11 @@ __device__ void decode_i2s_to_i8s(T1 *_i2s, T2 *_i8s, const int N = 16) static constexpr uint I4s_TO_I8s_MAGIC_NUM = 0x00000000; #pragma unroll - for (int i = 0; i < (N / 4); i++) + for (int i = 0; i < (N/4); i++) { asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(i8s[i]) - : "r"(i2s >> (2 * i)), "n"(BOTTOM_MASK), "n"(I4s_TO_I8s_MAGIC_NUM), "n"(immLut)); + : "r"(i2s >> (2*i)), "n"(BOTTOM_MASK), "n"(I4s_TO_I8s_MAGIC_NUM), "n"(immLut)); i8s[i] = __vsubss4(i8s[i], 0x02020202); } } @@ -92,7 +92,7 @@ __device__ void decode_i2s_to_i8s(T1 *_i2s, T2 *_i8s, const int N = 16) /** * @brief Single-row ternary GEMM kernel (M = 1). * - * Computes one row of @c dtype_transform[0,:] = (A * B^T) / s[0] * ws, + * Computes one row of @c dtype_transform[0,:] = (A * B^T)/s[0]*ws, * with @c A in int8, @c B packed as int2 (decoded on the fly), accumulated * in int32 via @c __dp4a, then scaled back to bf16. The output bias * @c ws is applied per @c ws_num column groups. @@ -118,19 +118,19 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel(int8_t* __restric in_thread_C_local[0] = 0; #pragma unroll for (int k_0 = 0; k_0 < K/(K_per_loop * K_block_size); ++k_0) { - *(int4*)(A_local + 0) = *(int4*)(A + ((k_0 * K_per_loop * K_block_size) + (((int)threadIdx.x) * K_per_loop))); + *(int4*)(A_local+0) = *(int4*)(A+((k_0*K_per_loop * K_block_size)+(((int)threadIdx.x)*K_per_loop))); B_reshape_local[0] = *(int*)(B + - (((int)blockIdx.x) * N_block_size * K / 4) + - (k_0 * K_block_size * K_per_loop * wmma_N / 4) + - ((((int)threadIdx.x) >> 1) * wmma_K * wmma_N / 4) + - ((((int)threadIdx.y) >> 3) * (wmma_K * wmma_N / 2) / 4) + - ((((int)threadIdx.x) & 1) * (wmma_K * wmma_N / 4) / 4) + - ((((int)threadIdx.y) & 7) * (wmma_K / 2) / 4) + (((int)blockIdx.x)*N_block_size * K/4) + + (k_0*K_block_size * K_per_loop * wmma_N/4) + + ((((int)threadIdx.x) >> 1)*wmma_K * wmma_N/4) + + ((((int)threadIdx.y) >> 3)*(wmma_K * wmma_N/2)/4) + + ((((int)threadIdx.x) & 1)*(wmma_K * wmma_N/4)/4) + + ((((int)threadIdx.y) & 7)*(wmma_K/2)/4) ); decode_i2s_to_i8s(B_reshape_local, B_decode_local, 16); #pragma unroll for (int k_2_0 = 0; k_2_0 < 4; ++k_2_0) { - in_thread_C_local[0] = __dp4a(*(int *)&A_local[((k_2_0 * 4))],*(int *)&B_decode_local[((k_2_0 * 4))], in_thread_C_local[0]); + in_thread_C_local[0] = __dp4a(*(int *)&A_local[((k_2_0*4))],*(int *)&B_decode_local[((k_2_0*4))], in_thread_C_local[0]); } } red_buf0[0] = in_thread_C_local[0]; @@ -138,17 +138,17 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel(int8_t* __restric for (int offset = K_block_size/2; offset > 0; offset /= 2) { red_buf0[0] += __shfl_down_sync(__activemask(), red_buf0[0], offset, K_block_size); } - int out_idx = ((((int)blockIdx.x) * N_block_size) + ((int)threadIdx.y)); - int ws_idx = out_idx / (N / ws_num); + int out_idx = ((((int)blockIdx.x)*N_block_size)+((int)threadIdx.y)); + int ws_idx = out_idx/(N/ws_num); if (threadIdx.x == 0) - dtype_transform[out_idx] = __float2bfloat16(((float)red_buf0[0]) / s[0] * ws[ws_idx]); + dtype_transform[out_idx] = __float2bfloat16(((float)red_buf0[0])/s[0]*ws[ws_idx]); } /** * @brief Multi-row ternary GEMM kernel (M > 1) using @c wmma fragments. * * Tiles M rows in chunks of @p M_ROWS per CTA. Each warp processes - * @c TILES_PER_WARP = @p M_ROWS / 64 row-tiles, accumulating int32 via + * @c TILES_PER_WARP = @p M_ROWS/64 row-tiles, accumulating int32 via * @c wmma::mma_sync against B fragments decoded from the i2 pack into * shared memory. * @@ -159,7 +159,7 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel(int8_t* __restric * @param A,B Activation (int8) and weight (int2-packed int8) buffers. * @param out bf16 output (M x N). * @param s,ws Per-row activation scale and per-column-group output scale. - * @param M Actual row count (may be < blockDim.y * M_ROWS - tail + * @param M Actual row count (may be < blockDim.y*M_ROWS-tail * threads predicate against this). */ template @@ -172,13 +172,13 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( constexpr int N_block_size = 16; constexpr int K_per_loop = 16, wmma_K = 32, wmma_N = 16; constexpr int K_CHUNK = 128; - constexpr int TILES_PER_WARP = M_ROWS / (4 * 16); + constexpr int TILES_PER_WARP = M_ROWS/(4*16); const int tx = (int)threadIdx.x; const int ty = (int)threadIdx.y; - const int tid = ty * 8 + tx; + const int tid = ty*8+tx; const int warp = tid >> 5; - const int m_base = (int)blockIdx.y * M_ROWS; + const int m_base = (int)blockIdx.y*M_ROWS; __shared__ signed char A_smem[M_ROWS][K_CHUNK]; __shared__ signed char W_smem[16][K_CHUNK]; @@ -194,33 +194,33 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( for (int t = 0; t < TILES_PER_WARP; ++t) wmma::fill_fragment(acc[t], 0); - for (int k_0 = 0; k_0 < K / K_CHUNK; ++k_0) { + for (int k_0 = 0; k_0 < K/K_CHUNK; ++k_0) { #pragma unroll - for (int r = 0; r < (M_ROWS * K_CHUNK / 16) / 128; ++r) { - const int idx = tid + r * 128; + for (int r = 0; r < (M_ROWS * K_CHUNK/16)/128; ++r) { + const int idx = tid+r*128; const int m = idx >> 3; - const int kk = (idx & 7) * 16; - const int mrow = m_base + m; - const int8_t* aptr = A + ((mrow < M ? mrow : 0) * K) + k_0 * K_CHUNK + kk; + const int kk = (idx & 7)*16; + const int mrow = m_base+m; + const int8_t* aptr = A+((mrow < M ? mrow : 0)*K)+k_0*K_CHUNK+kk; *(int4*)(&A_smem[m][kk]) = *(const int4*)aptr; } B_reshape_local[0] = *(int*)(B + - (((int)blockIdx.x) * N_block_size * K / 4) + - (k_0 * 8 * K_per_loop * wmma_N / 4) + - ((tx >> 1) * wmma_K * wmma_N / 4) + - ((ty >> 3) * (wmma_K * wmma_N / 2) / 4) + - ((tx & 1) * (wmma_K * wmma_N / 4) / 4) + - ((ty & 7) * (wmma_K / 2) / 4)); + (((int)blockIdx.x)*N_block_size * K/4) + + (k_0*8*K_per_loop * wmma_N/4) + + ((tx >> 1)*wmma_K * wmma_N/4) + + ((ty >> 3)*(wmma_K * wmma_N/2)/4) + + ((tx & 1)*(wmma_K * wmma_N/4)/4) + + ((ty & 7)*(wmma_K/2)/4)); decode_i2s_to_i8s(B_reshape_local, B_decode_local, 16); - *(int4*)(&W_smem[ty][tx * 16]) = *(int4*)(&B_decode_local[0]); + *(int4*)(&W_smem[ty][tx*16]) = *(int4*)(&B_decode_local[0]); __syncthreads(); #pragma unroll - for (int k16 = 0; k16 < K_CHUNK / 16; ++k16) { - wmma::load_matrix_sync(b_frag, &W_smem[0][k16 * 16], K_CHUNK); + for (int k16 = 0; k16 < K_CHUNK/16; ++k16) { + wmma::load_matrix_sync(b_frag, &W_smem[0][k16*16], K_CHUNK); #pragma unroll for (int t = 0; t < TILES_PER_WARP; ++t) { - wmma::load_matrix_sync(a_frag[t], &A_smem[warp * 16 + t * 64][k16 * 16], K_CHUNK); + wmma::load_matrix_sync(a_frag[t], &A_smem[warp*16+t*64][k16*16], K_CHUNK); wmma::mma_sync(acc[t], a_frag[t], b_frag, acc[t]); } } @@ -229,20 +229,20 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( #pragma unroll for (int t = 0; t < TILES_PER_WARP; ++t) - wmma::store_matrix_sync(&C_smem[warp * 16 + t * 64][0], acc[t], 16, wmma::mem_row_major); + wmma::store_matrix_sync(&C_smem[warp*16+t*64][0], acc[t], 16, wmma::mem_row_major); __syncthreads(); - const int n_base = ((int)blockIdx.x) * N_block_size; - const int ws_idx = n_base / (N / ws_num); + const int n_base = ((int)blockIdx.x)*N_block_size; + const int ws_idx = n_base/(N/ws_num); const float wsv = ws[ws_idx]; #pragma unroll - for (int e = 0; e < (M_ROWS * 16) / 128; ++e) { - const int lin = tid + e * 128; + for (int e = 0; e < (M_ROWS*16)/128; ++e) { + const int lin = tid+e*128; const int ml = lin >> 4; const int col = lin & 15; - const int m = m_base + ml; + const int m = m_base+ml; if (m < M) - out[m * N + n_base + col] = __float2bfloat16(((float)C_smem[ml][col]) / s[m] * wsv); + out[m * N+n_base+col] = __float2bfloat16(((float)C_smem[ml][col])/s[m]*wsv); } } @@ -257,7 +257,7 @@ static inline void launch_ladder_int8xint2_m( int8_t* A, int8_t* B, __nv_bfloat16* out, float* s, float* ws, int M, cudaStream_t stream) { ladder_int8xint2_kernel_m - <<>>(A, B, out, s, ws, M); + <<>>(A, B, out, s, ws, M); } /** @@ -292,29 +292,29 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( constexpr int K_per_loop = 16, wmma_K = 32, wmma_N = 16; constexpr int K_CHUNK = 128; constexpr int WARPS = 4; - constexpr int M_TILES = M_ROWS / 16; - constexpr int N_BLOCKS = N / 16; // total 16-wide column tiles in the matrix + constexpr int M_TILES = M_ROWS/16; + constexpr int N_BLOCKS = N/16; // total 16-wide column tiles in the matrix // Warps split two ways. N_TILES of them take different column tiles (that is // the A-reuse win); the remaining WARPS/N_TILES take different row ranges // (that is parallelism, which matters when N is small enough that column // tiles alone cannot fill the GPU). N_TILES == 1 reproduces the original // kernel's mapping exactly. - constexpr int M_GROUPS = WARPS / N_TILES; - constexpr int M_PER_WARP = M_TILES / M_GROUPS; + constexpr int M_GROUPS = WARPS/N_TILES; + constexpr int M_PER_WARP = M_TILES/M_GROUPS; // Row stride padded to break shared-memory bank conflicts: at a stride of // 128 B every row of a 16-row fragment starts on bank 0, so each // load_matrix_sync serialises 16 ways. 144 B (still a multiple of the 16 B // that wmma requires for integer ldm) spreads them over 8 banks. - constexpr int SM_STRIDE = K_CHUNK + 16; + constexpr int SM_STRIDE = K_CHUNK+16; const int tx = (int)threadIdx.x; // 0..7 const int ty = (int)threadIdx.y; // 0..15 - const int tid = ty * 8 + tx; // 0..127 + const int tid = ty*8+tx; // 0..127 const int warp = tid >> 5; // 0..3 const int lane = tid & 31; - const int m_base = (int)blockIdx.y * M_ROWS; + const int m_base = (int)blockIdx.y*M_ROWS; __shared__ signed char A_smem[M_ROWS][SM_STRIDE]; __shared__ signed char W_smem[N_TILES][16][SM_STRIDE]; @@ -323,9 +323,9 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( // Column tile this warp owns. N is not always a multiple of 16*N_TILES // (the ViT's 4304 is 269 tiles), so tiles past the end are skipped rather // than clamped -- clamping would double-write real columns. - const int my_tile = (int)blockIdx.x * N_TILES + (warp % N_TILES); + const int my_tile = (int)blockIdx.x*N_TILES+(warp%N_TILES); const bool my_tile_valid = my_tile < N_BLOCKS; - const int m_tile_base = (warp / N_TILES) * M_PER_WARP; + const int m_tile_base = (warp/N_TILES)*M_PER_WARP; int B_reshape_local[1]; signed char B_decode_local[K_per_loop]; @@ -337,14 +337,14 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( for (int t = 0; t < M_PER_WARP; ++t) wmma::fill_fragment(acc[t], 0); - for (int k_0 = 0; k_0 < K / K_CHUNK; ++k_0) { + for (int k_0 = 0; k_0 < K/K_CHUNK; ++k_0) { #pragma unroll - for (int r = 0; r < (M_ROWS * K_CHUNK / 16) / 128; ++r) { - const int idx = tid + r * 128; + for (int r = 0; r < (M_ROWS * K_CHUNK/16)/128; ++r) { + const int idx = tid+r*128; const int m = idx >> 3; - const int kk = (idx & 7) * 16; - const int mrow = m_base + m; - const int8_t* aptr = A + ((mrow < M ? mrow : 0) * K) + k_0 * K_CHUNK + kk; + const int kk = (idx & 7)*16; + const int mrow = m_base+m; + const int8_t* aptr = A+((mrow < M ? mrow : 0)*K)+k_0*K_CHUNK+kk; *(int4*)(&A_smem[m][kk]) = *(const int4*)aptr; } @@ -352,28 +352,28 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( // pack's swizzle exactly; only the tile base changes per j. #pragma unroll for (int j = 0; j < N_TILES; ++j) { - const int tile = (int)blockIdx.x * N_TILES + j; + const int tile = (int)blockIdx.x*N_TILES+j; if (tile < N_BLOCKS) { B_reshape_local[0] = *(int*)(B + - ((size_t)tile * 16 * K / 4) + - (k_0 * 8 * K_per_loop * wmma_N / 4) + - ((tx >> 1) * wmma_K * wmma_N / 4) + - ((ty >> 3) * (wmma_K * wmma_N / 2) / 4) + - ((tx & 1) * (wmma_K * wmma_N / 4) / 4) + - ((ty & 7) * (wmma_K / 2) / 4)); + ((size_t)tile*16*K/4) + + (k_0*8*K_per_loop * wmma_N/4) + + ((tx >> 1)*wmma_K * wmma_N/4) + + ((ty >> 3)*(wmma_K * wmma_N/2)/4) + + ((tx & 1)*(wmma_K * wmma_N/4)/4) + + ((ty & 7)*(wmma_K/2)/4)); decode_i2s_to_i8s(B_reshape_local, B_decode_local, 16); - *(int4*)(&W_smem[j][ty][tx * 16]) = *(int4*)(&B_decode_local[0]); + *(int4*)(&W_smem[j][ty][tx*16]) = *(int4*)(&B_decode_local[0]); } } __syncthreads(); if (my_tile_valid) { #pragma unroll - for (int k16 = 0; k16 < K_CHUNK / 16; ++k16) { - wmma::load_matrix_sync(b_frag, &W_smem[warp % N_TILES][0][k16 * 16], SM_STRIDE); + for (int k16 = 0; k16 < K_CHUNK/16; ++k16) { + wmma::load_matrix_sync(b_frag, &W_smem[warp%N_TILES][0][k16*16], SM_STRIDE); #pragma unroll for (int t = 0; t < M_PER_WARP; ++t) { - wmma::load_matrix_sync(a_frag, &A_smem[(m_tile_base + t) * 16][k16 * 16], SM_STRIDE); + wmma::load_matrix_sync(a_frag, &A_smem[(m_tile_base+t)*16][k16*16], SM_STRIDE); wmma::mma_sync(acc[t], a_frag, b_frag, acc[t]); } } @@ -384,8 +384,8 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( if (!my_tile_valid) return; - const int n_base = my_tile * 16; - const float wsv = ws[n_base / (N / ws_num)]; + const int n_base = my_tile*16; + const float wsv = ws[n_base/(N/ws_num)]; // One row-tile at a time through a per-warp staging buffer: a full // M_ROWS x 16 int32 buffer per warp would cost more shared memory than the @@ -395,14 +395,14 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( wmma::store_matrix_sync(&C_smem[warp][0][0], acc[t], 16, wmma::mem_row_major); __syncwarp(); #pragma unroll - for (int e = 0; e < (16 * 16) / 32; ++e) { - const int lin = lane + e * 32; + for (int e = 0; e < (16*16)/32; ++e) { + const int lin = lane+e*32; const int ml = lin >> 4; const int col = lin & 15; - const int m = m_base + (m_tile_base + t) * 16 + ml; + const int m = m_base+(m_tile_base+t)*16+ml; if (m < M) - out[(size_t)m * N + n_base + col] = - __float2bfloat16(((float)C_smem[warp][ml][col]) / s[m] * wsv); + out[(size_t)m * N+n_base+col] = + __float2bfloat16(((float)C_smem[warp][ml][col])/s[m]*wsv); } __syncwarp(); } @@ -441,9 +441,9 @@ template static inline void launch_ladder_int8xint2_m_wide( int8_t* A, int8_t* B, __nv_bfloat16* out, float* s, float* ws, int M, cudaStream_t stream) { - constexpr int N_BLOCKS = N / 16; + constexpr int N_BLOCKS = N/16; ladder_int8xint2_kernel_m_wide - <<>>(A, B, out, s, ws, M); } @@ -472,7 +472,7 @@ __global__ void act_quant_kernel( const int m = (int)blockIdx.x; const int tid = (int)threadIdx.x; const __nv_bfloat16* row_in = in + m * K; - int8_t* row_out = out + m * K; + int8_t* row_out = out+m * K; float local_max = 0.0f; for (int k = tid; k < K; k += BLOCK_THREADS) { @@ -494,7 +494,7 @@ __global__ void act_quant_kernel( smem[warp_id] = local_max; __syncthreads(); if (warp_id == 0) { - float v = (tid < (BLOCK_THREADS + 31) / 32) ? smem[lane] : 0.0f; + float v = (tid < (BLOCK_THREADS+31)/32) ? smem[lane] : 0.0f; for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); if (other > v) @@ -505,12 +505,12 @@ __global__ void act_quant_kernel( } __syncthreads(); const float amax = smem[0] < 1e-5f ? 1e-5f : smem[0]; - const float scale = 127.0f / amax; + const float scale = 127.0f/amax; if (tid == 0) scales[m] = scale; for (int k = tid; k < K; k += BLOCK_THREADS) { - float v = __bfloat162float(row_in[k]) * scale; + float v = __bfloat162float(row_in[k])*scale; float q = nearbyintf(v); if (q > 127.0f) q = 127.0f; diff --git a/src/kernels/bitvla/bitvla_fp32head_cuda.cu b/src/kernels/bitvla/bitvla_fp32head_cuda.cu index 0dbb6c9..d40c20d 100644 --- a/src/kernels/bitvla/bitvla_fp32head_cuda.cu +++ b/src/kernels/bitvla/bitvla_fp32head_cuda.cu @@ -72,15 +72,15 @@ static float* upload_f32(const float* h, size_t n) { } __global__ void gelu_erf_fp32_kernel(const float* __restrict__ in, float* __restrict__ out, int N) { - const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int i = blockIdx.x*blockDim.x+threadIdx.x; if (i >= N) return; float x = in[i]; - out[i] = 0.5f * x * (1.0f + erff(x * 0.70710678118654752440f)); + out[i] = 0.5f * x * (1.0f+erff(x*0.70710678118654752440f)); } __global__ void relu_fp32_kernel(float* __restrict__ inout, int N) { - const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int i = blockIdx.x*blockDim.x+threadIdx.x; if (i >= N) return; float v = inout[i]; @@ -96,8 +96,8 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, float eps, int K) { const int m = blockIdx.x; const int tid = threadIdx.x; - const float* row = x + (size_t)m * K; - float* o = out + (size_t)m * K; + const float* row = x+(size_t)m * K; + float* o = out+(size_t)m * K; float sum = 0.0f; for (int k = tid; k < K; k += BLOCK) @@ -109,18 +109,18 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, smem[tid >> 5] = sum; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float mean = smem[0] / (float)K; + const float mean = smem[0]/(float)K; float vsum = 0.0f; for (int k = tid; k < K; k += BLOCK) { - float v = row[k] - mean; + float v = row[k]-mean; vsum += v * v; } for (int off = 16; off > 0; off >>= 1) @@ -129,35 +129,35 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float inv_std = rsqrtf(smem[0] / (float)K + eps); + const float inv_std = rsqrtf(smem[0]/(float)K+eps); for (int k = tid; k < K; k += BLOCK) { - o[k] = (row[k] - mean) * inv_std * w[k] + b[k]; + o[k] = (row[k]-mean)*inv_std * w[k]+b[k]; } } __global__ void add_bias_fp32_kernel(const float* x, const float* bias, float* out, int M, int K) { const int m = blockIdx.x; - const int k = blockIdx.y * blockDim.x + threadIdx.x; + const int k = blockIdx.y*blockDim.x+threadIdx.x; if (k >= K) return; - const size_t i = (size_t)m * K + k; - out[i] = x[i] + bias[k]; + const size_t i = (size_t)m * K+k; + out[i] = x[i]+bias[k]; } __global__ void add_fp32_kernel(const float* a, const float* b, float* out, int N) { - const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int i = blockIdx.x*blockDim.x+threadIdx.x; if (i >= N) return; - out[i] = a[i] + b[i]; + out[i] = a[i]+b[i]; } static int linear_bias_fp32( @@ -181,7 +181,7 @@ static int linear_bias_fp32( } if (bias) { const int B = 256; - const int n_kb = (N_out + B - 1) / B; + const int n_kb = (N_out+B-1)/B; add_bias_fp32_kernel<<>>(out, bias, out, M, N_out); } return 0; @@ -246,8 +246,8 @@ extern "C" bitvla_fp32head_cuda_ctx* bitvla_fp32head_cuda_init( CUDA_OK_NULL(cudaMalloc(&ctx->d_pp_h1, (size_t)lm_hidden * sizeof(float))); CUDA_OK_NULL(cudaMalloc(&ctx->d_pp_out, (size_t)lm_hidden * sizeof(float))); - CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_in, (size_t)chunk * ctx->ah_in_dim * sizeof(float))); - CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_norm_big, (size_t)chunk * ctx->ah_in_dim * sizeof(float))); + CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_in, (size_t)chunk * ctx->ah_in_dim*sizeof(float))); + CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_norm_big, (size_t)chunk * ctx->ah_in_dim*sizeof(float))); CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_h, (size_t)chunk * lm_hidden * sizeof(float))); CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_tmp, (size_t)chunk * lm_hidden * sizeof(float))); CUDA_OK_NULL(cudaMalloc(&ctx->d_ah_tmp2, (size_t)chunk * lm_hidden * sizeof(float))); @@ -275,7 +275,7 @@ extern "C" int bitvla_fp32head_proprio_forward( cudaStream_t stream) { - CUDA_OK_RET(cudaMemcpyAsync(ctx->d_state, host_state, (size_t)ctx->proprio_dim * sizeof(float), + CUDA_OK_RET(cudaMemcpyAsync(ctx->d_state, host_state, (size_t)ctx->proprio_dim*sizeof(float), cudaMemcpyHostToDevice, stream)); if (linear_bias_fp32(ctx->cublas, stream, ctx->pp_fc1_w, ctx->pp_fc1_b, @@ -284,13 +284,13 @@ extern "C" int bitvla_fp32head_proprio_forward( { const int B = 256; const int N = ctx->lm_hidden; - gelu_erf_fp32_kernel<<>>(ctx->d_pp_h1, ctx->d_pp_h1, N); + gelu_erf_fp32_kernel<<>>(ctx->d_pp_h1, ctx->d_pp_h1, N); } if (linear_bias_fp32(ctx->cublas, stream, ctx->pp_fc2_w, ctx->pp_fc2_b, ctx->d_pp_h1, ctx->d_pp_out, 1, ctx->lm_hidden, ctx->lm_hidden) != 0) return -1; - CUDA_OK_RET(cudaMemcpyAsync(host_out, ctx->d_pp_out, (size_t)ctx->lm_hidden * sizeof(float), + CUDA_OK_RET(cudaMemcpyAsync(host_out, ctx->d_pp_out, (size_t)ctx->lm_hidden*sizeof(float), cudaMemcpyDeviceToHost, stream)); CUDA_OK_RET(cudaStreamSynchronize(stream)); return 0; @@ -322,7 +322,7 @@ extern "C" int bitvla_fp32head_action_forward( { const int B = 256; const int N = M * H; - relu_fp32_kernel<<>>(ctx->d_ah_h, N); + relu_fp32_kernel<<>>(ctx->d_ah_h, N); } { @@ -337,13 +337,13 @@ extern "C" int bitvla_fp32head_action_forward( { const int B = 256; const int N = M * H; - relu_fp32_kernel<<>>(ctx->d_ah_tmp2, N); + relu_fp32_kernel<<>>(ctx->d_ah_tmp2, N); } { const int B = 256; const int N = M * H; - add_fp32_kernel<<>>(ctx->d_ah_h, ctx->d_ah_tmp2, ctx->d_ah_h, N); + add_fp32_kernel<<>>(ctx->d_ah_h, ctx->d_ah_tmp2, ctx->d_ah_h, N); } { @@ -356,12 +356,12 @@ extern "C" int bitvla_fp32head_action_forward( { const int B = 256; const int N = M * H; - relu_fp32_kernel<<>>(ctx->d_ah_tmp2, N); + relu_fp32_kernel<<>>(ctx->d_ah_tmp2, N); } { const int B = 256; const int N = M * H; - add_fp32_kernel<<>>(ctx->d_ah_h, ctx->d_ah_tmp2, ctx->d_ah_h, N); + add_fp32_kernel<<>>(ctx->d_ah_h, ctx->d_ah_tmp2, ctx->d_ah_h, N); } { diff --git a/src/kernels/bitvla/bitvla_fp32head_cuda.h b/src/kernels/bitvla/bitvla_fp32head_cuda.h index 5f5dd58..dc6ccbc 100644 --- a/src/kernels/bitvla/bitvla_fp32head_cuda.h +++ b/src/kernels/bitvla/bitvla_fp32head_cuda.h @@ -14,7 +14,7 @@ /** * @file bitvla_fp32head_cuda.h - * @brief FP32 BitVLA action head + proprioception projection. + * @brief FP32 BitVLA action head+proprioception projection. * * BitVLA's LM and ViT are 1.58-bit ternary; its small action head and * proprio projector are kept in FP32 (cuBLAS GEMM) to preserve regression @@ -97,7 +97,7 @@ bitvla_fp32head_cuda_ctx* bitvla_fp32head_cuda_init( * @param ctx Context returned by @ref bitvla_fp32head_cuda_init. * @param host_state Length-@c proprio_dim FP32 input on the host. * @param host_out Length-@c lm_hidden FP32 output on the host. - * @param stream CUDA stream used for the H2D / D2H transfers. + * @param stream CUDA stream used for the H2D/D2H transfers. * @return 0 on success, non-zero on dispatch failure. */ int bitvla_fp32head_proprio_forward( @@ -113,7 +113,7 @@ int bitvla_fp32head_proprio_forward( * @param host_norm_actions Length-@c chunk*action_dim FP32 output on the * host, normalised to training statistics. The caller un-normalises * to world units. - * @param stream CUDA stream used for the H2D / D2H transfers. + * @param stream CUDA stream used for the H2D/D2H transfers. * @return 0 on success, non-zero on dispatch failure. */ int bitvla_fp32head_action_forward( diff --git a/src/kernels/bitvla/bitvla_lm_cuda.cu b/src/kernels/bitvla/bitvla_lm_cuda.cu index 2ff5401..2e0f193 100644 --- a/src/kernels/bitvla/bitvla_lm_cuda.cu +++ b/src/kernels/bitvla/bitvla_lm_cuda.cu @@ -42,8 +42,8 @@ __global__ void rmsnorm_bf16_kernel(const __nv_bfloat16* __restrict__ x, { const int m = (int)blockIdx.x; const int tid = (int)threadIdx.x; - const __nv_bfloat16* row = x + m * K; - __nv_bfloat16* o = out + m * K; + const __nv_bfloat16* row = x+m * K; + __nv_bfloat16* o = out+m * K; float ss = 0.0f; for (int k = tid; k < K; k += BLOCK) { @@ -57,18 +57,18 @@ __global__ void rmsnorm_bf16_kernel(const __nv_bfloat16* __restrict__ x, smem[tid >> 5] = ss; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float mean = smem[0] / (float)K; - const float scale = rsqrtf(mean + eps); + const float mean = smem[0]/(float)K; + const float scale = rsqrtf(mean+eps); for (int k = tid; k < K; k += BLOCK) { - float v = __bfloat162float(row[k]) * scale; + float v = __bfloat162float(row[k])*scale; float wv = __bfloat162float(w[k]); o[k] = __float2bfloat16(v * wv); } @@ -83,17 +83,17 @@ __global__ void rope_neox_bf16_kernel(__nv_bfloat16* __restrict__ inout, const int h = (int)blockIdx.x; const int s = (int)blockIdx.y; const int tid = (int)threadIdx.x; - const int half = D / 2; - __nv_bfloat16* row = inout + ((size_t)h * S + s) * D; - const float* c_row = cos_tab + (size_t)s * half; - const float* s_row = sin_tab + (size_t)s * half; + const int half = D/2; + __nv_bfloat16* row = inout+((size_t)h * S+s)*D; + const float* c_row = cos_tab+(size_t)s * half; + const float* s_row = sin_tab+(size_t)s * half; for (int k = tid; k < half; k += BLOCK) { float c = c_row[k]; float si= s_row[k]; float a = __bfloat162float(row[k]); - float b = __bfloat162float(row[k + half]); - row[k] = __float2bfloat16(a * c - b * si); - row[k + half] = __float2bfloat16(b * c + a * si); + float b = __bfloat162float(row[k+half]); + row[k] = __float2bfloat16(a * c-b * si); + row[k+half] = __float2bfloat16(b * c+a * si); } } @@ -103,11 +103,11 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, { const int row = (int)blockIdx.x; const int tid = (int)threadIdx.x; - __nv_bfloat16* r = inout + (size_t)row * S; + __nv_bfloat16* r = inout+(size_t)row * S; float mx = -INFINITY; for (int i = tid; i < S; i += BLOCK) { - float v = __bfloat162float(r[i]) * scale; + float v = __bfloat162float(r[i])*scale; if (v > mx) mx = v; } @@ -121,7 +121,7 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, smem[tid >> 5] = mx; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : -INFINITY; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : -INFINITY; for (int off = 16; off > 0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); if (other > v) @@ -135,7 +135,7 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, float s_sum = 0.0f; for (int i = tid; i < S; i += BLOCK) { - s_sum += expf(__bfloat162float(r[i]) * scale - max_v); + s_sum += expf(__bfloat162float(r[i])*scale-max_v); } for (int off = 16; off > 0; off >>= 1) s_sum += __shfl_down_sync(0xffffffff, s_sum, off); @@ -143,17 +143,17 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, smem[tid >> 5] = s_sum; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float inv_sum = 1.0f / smem[0]; + const float inv_sum = 1.0f/smem[0]; for (int i = tid; i < S; i += BLOCK) { - float v = expf(__bfloat162float(r[i]) * scale - max_v) * inv_sum; + float v = expf(__bfloat162float(r[i])*scale-max_v)*inv_sum; r[i] = __float2bfloat16(v); } } @@ -161,7 +161,7 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, __global__ void squared_relu_mul_bf16_kernel(const __nv_bfloat16* g, const __nv_bfloat16* u, __nv_bfloat16* out, int N) { - const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); + const int i = (int)(blockIdx.x*blockDim.x+threadIdx.x); if (i >= N) return; float gv = __bfloat162float(g[i]); @@ -172,10 +172,10 @@ __global__ void squared_relu_mul_bf16_kernel(const __nv_bfloat16* g, __global__ void add_bf16_kernel(const __nv_bfloat16* a, const __nv_bfloat16* b, __nv_bfloat16* out, int N) { - const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); + const int i = (int)(blockIdx.x*blockDim.x+threadIdx.x); if (i >= N) return; - out[i] = __float2bfloat16(__bfloat162float(a[i]) + __bfloat162float(b[i])); + out[i] = __float2bfloat16(__bfloat162float(a[i])+__bfloat162float(b[i])); } __global__ void repeat_kv_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* out, @@ -183,9 +183,9 @@ __global__ void repeat_kv_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* ou const int q_h = (int)blockIdx.x; const int s = (int)blockIdx.y; const int tid = (int)threadIdx.x; - const int kv_h = q_h * n_kv / n_q; + const int kv_h = q_h * n_kv/n_q; for (int k = tid; k < hd; k += blockDim.x) { - out[((size_t)q_h * seq + s) * hd + k] = in[((size_t)kv_h * seq + s) * hd + k]; + out[((size_t)q_h * seq+s)*hd+k] = in[((size_t)kv_h * seq+s)*hd+k]; } } @@ -196,7 +196,7 @@ __global__ void transpose_sHhd_to_HShd_bf16_kernel(const __nv_bfloat16* in, const int h = (int)blockIdx.x; const int tid = (int)threadIdx.x; for (int k = tid; k < hd; k += blockDim.x) { - out[((size_t)h * S + s) * hd + k] = in[((size_t)s * H + h) * hd + k]; + out[((size_t)h * S+s)*hd+k] = in[((size_t)s * H+h)*hd+k]; } } @@ -207,7 +207,7 @@ __global__ void transpose_HShd_to_sHhd_bf16_kernel(const __nv_bfloat16* in, const int s = (int)blockIdx.y; const int tid = (int)threadIdx.x; for (int k = tid; k < hd; k += blockDim.x) { - out[((size_t)s * H + h) * hd + k] = in[((size_t)h * S + s) * hd + k]; + out[((size_t)s * H+h)*hd+k] = in[((size_t)h * S+s)*hd+k]; } } @@ -219,7 +219,7 @@ __global__ void gather_rows_bf16_kernel(const __nv_bfloat16* in, const int r = row_ids[m]; const int tid = (int)threadIdx.x; for (int k = tid; k < K; k += blockDim.x) { - out[(size_t)m * K + k] = in[(size_t)r * K + k]; + out[(size_t)m * K+k] = in[(size_t)r * K+k]; } } @@ -243,12 +243,12 @@ extern "C" void bitvla_softmax_scaled_bf16(__nv_bfloat16* inout, float scale, extern "C" void bitvla_squared_relu_mul_bf16(const __nv_bfloat16* g, const __nv_bfloat16* u, __nv_bfloat16* out, int N, cudaStream_t stream) { constexpr int B = 256; - squared_relu_mul_bf16_kernel<<>>(g, u, out, N); + squared_relu_mul_bf16_kernel<<>>(g, u, out, N); } extern "C" void bitvla_add_bf16(const __nv_bfloat16* a, const __nv_bfloat16* b, __nv_bfloat16* out, int N, cudaStream_t stream) { constexpr int B = 256; - add_bf16_kernel<<>>(a, b, out, N); + add_bf16_kernel<<>>(a, b, out, N); } extern "C" void bitvla_repeat_kv_bf16(const __nv_bfloat16* in, __nv_bfloat16* out, int n_q, int n_kv, int seq, int hd, cudaStream_t stream) { @@ -330,27 +330,27 @@ extern "C" bitvla_lm_cuda_ctx* bitvla_lm_cuda_init(int hidden, int n_q, int n_kv return nullptr; } - const int half = head_dim / 2; + const int half = head_dim/2; std::vector h_cos((size_t)max_seq * half), h_sin((size_t)max_seq * half); for (int s = 0; s < max_seq; ++s) { for (int k = 0; k < half; ++k) { - float freq = 1.0f / std::pow(rope_base, (float)(2 * k) / (float)head_dim); + float freq = 1.0f/std::pow(rope_base, (float)(2*k)/(float)head_dim); float ang = (float)s * freq; - h_cos[(size_t)s * half + k] = std::cos(ang); - h_sin[(size_t)s * half + k] = std::sin(ang); + h_cos[(size_t)s * half+k] = std::cos(ang); + h_sin[(size_t)s * half+k] = std::sin(ang); } } CUDA_OKV(cudaMalloc(&ctx->d_cos, (size_t)max_seq * half * sizeof(float))); CUDA_OKV(cudaMalloc(&ctx->d_sin, (size_t)max_seq * half * sizeof(float))); - CUDA_OKV(cudaMemcpy(ctx->d_cos, h_cos.data(), h_cos.size() * sizeof(float), cudaMemcpyHostToDevice)); - CUDA_OKV(cudaMemcpy(ctx->d_sin, h_sin.data(), h_sin.size() * sizeof(float), cudaMemcpyHostToDevice)); + CUDA_OKV(cudaMemcpy(ctx->d_cos, h_cos.data(), h_cos.size()*sizeof(float), cudaMemcpyHostToDevice)); + CUDA_OKV(cudaMemcpy(ctx->d_sin, h_sin.data(), h_sin.size()*sizeof(float), cudaMemcpyHostToDevice)); const size_t bf16 = sizeof(__nv_bfloat16); CUDA_OKV(cudaMalloc(&ctx->d_h, (size_t)max_seq * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_h_norm, (size_t)max_seq * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_act_int8_h, (size_t)max_seq * hidden)); CUDA_OKV(cudaMalloc(&ctx->d_act_s, (size_t)max_seq * sizeof(float))); - CUDA_OKV(cudaMalloc(&ctx->d_qkv, (size_t)max_seq * (ctx->hidden_q + 2 * ctx->hidden_kv) * bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_qkv, (size_t)max_seq * (ctx->hidden_q+2*ctx->hidden_kv)*bf16)); CUDA_OKV(cudaMalloc(&ctx->d_q_HShd, (size_t)n_q * max_seq * head_dim * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_k_HShd, (size_t)n_kv * max_seq * head_dim * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_v_HShd, (size_t)n_kv * max_seq * head_dim * bf16)); @@ -361,7 +361,7 @@ extern "C" bitvla_lm_cuda_ctx* bitvla_lm_cuda_init(int hidden, int n_q, int n_kv CUDA_OKV(cudaMalloc(&ctx->d_attn_merged, (size_t)max_seq * ctx->hidden_q * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_o_out, (size_t)max_seq * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_act_int8_ffn,(size_t)max_seq * ffn)); - CUDA_OKV(cudaMalloc(&ctx->d_gate_up, (size_t)max_seq * 2 * ffn * bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_gate_up, (size_t)max_seq*2*ffn * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_gate_sq_up, (size_t)max_seq * ffn * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_down_out, (size_t)max_seq * hidden * bf16)); return ctx; @@ -425,15 +425,15 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea bitvla_act_quant_cuda(ctx->d_h_norm, ctx->d_act_int8_h, ctx->d_act_s, seq, hidden, stream); __nv_bfloat16* q_dense = ctx->d_qkv; - __nv_bfloat16* k_dense = ctx->d_qkv + (size_t)seq * hq; - __nv_bfloat16* v_dense = ctx->d_qkv + (size_t)seq * (hq + hkv); + __nv_bfloat16* k_dense = ctx->d_qkv+(size_t)seq * hq; + __nv_bfloat16* v_dense = ctx->d_qkv+(size_t)seq * (hq+hkv); bitlinear_int8xint2_m(ctx->d_act_int8_h, lr.q_packed, q_dense, ctx->d_act_s, lr.q_ws, seq, hq, hidden, stream); bitlinear_int8xint2_m(ctx->d_act_int8_h, lr.k_packed, k_dense, ctx->d_act_s, lr.k_ws, seq, hkv, hidden, stream); bitlinear_int8xint2_m(ctx->d_act_int8_h, lr.v_packed, v_dense, ctx->d_act_s, lr.v_ws, seq, hkv, hidden, stream); - l0_dump("L0_02_qkv_proj", ctx->d_qkv, (size_t)seq * (hq + 2*hkv)); + l0_dump("L0_02_qkv_proj", ctx->d_qkv, (size_t)seq * (hq+2*hkv)); // Split the interleaved [seq, H*hd] projections into head-major [H, seq, hd] // with one kernel per tensor instead of a cudaMemcpy2DAsync per head (30 tiny @@ -465,7 +465,7 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea return -1; } - const float scl = 1.0f / std::sqrt((float)hd); + const float scl = 1.0f/std::sqrt((float)hd); bitvla_softmax_scaled_bf16(ctx->d_scores, scl, n_q * seq, seq, stream); cbs = cublasGemmStridedBatchedEx( @@ -504,7 +504,7 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea bitvla_act_quant_cuda(ctx->d_h_norm, ctx->d_act_int8_h, ctx->d_act_s, seq, hidden, stream); bitlinear_int8xint2_m(ctx->d_act_int8_h, lr.gate_up_packed, ctx->d_gate_up, - ctx->d_act_s, lr.gate_up_ws, seq, 2 * ffn, hidden, stream); + ctx->d_act_s, lr.gate_up_ws, seq, 2*ffn, hidden, stream); gate_up_fused_sqrelu_mul_bf16(ctx->d_gate_up, ctx->d_gate_sq_up, seq, ffn, stream); @@ -527,15 +527,15 @@ static int run_layer(bitvla_lm_cuda_ctx* ctx, int L, int seq, cudaStream_t strea __global__ void gate_up_fused_sqrelu_mul_bf16_kernel(const __nv_bfloat16* __restrict__ gu, __nv_bfloat16* __restrict__ out, int seq, int ffn) { - const int idx = (int)(blockIdx.x * blockDim.x + threadIdx.x); + const int idx = (int)(blockIdx.x*blockDim.x+threadIdx.x); const int total = seq * ffn; if (idx >= total) return; - const int s = idx / ffn; - const int k = idx % ffn; - const size_t row_base = (size_t)s * 2 * ffn; - float g = __bfloat162float(gu[row_base + k]); - float u = __bfloat162float(gu[row_base + ffn + k]); + const int s = idx/ffn; + const int k = idx%ffn; + const size_t row_base = (size_t)s*2*ffn; + float g = __bfloat162float(gu[row_base+k]); + float u = __bfloat162float(gu[row_base+ffn+k]); if (g < 0.0f) g = 0.0f; out[(size_t)idx] = __float2bfloat16(g * g * u); @@ -544,7 +544,7 @@ extern "C" void gate_up_fused_sqrelu_mul_bf16(const __nv_bfloat16* gu, __nv_bflo int seq, int ffn, cudaStream_t stream) { const int total = seq * ffn; const int B = 256; - gate_up_fused_sqrelu_mul_bf16_kernel<<>>(gu, out, seq, ffn); } @@ -557,8 +557,8 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, { const int m = (int)blockIdx.x; const int tid = (int)threadIdx.x; - const __nv_bfloat16* row = x + (size_t)m * K; - __nv_bfloat16* o = out + (size_t)m * K; + const __nv_bfloat16* row = x+(size_t)m * K; + __nv_bfloat16* o = out+(size_t)m * K; float sum = 0.0f; for (int k = tid; k < K; k += BLOCK) @@ -570,18 +570,18 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, smem[tid >> 5] = sum; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float mean = smem[0] / (float)K; + const float mean = smem[0]/(float)K; float vsum = 0.0f; for (int k = tid; k < K; k += BLOCK) { - float v = __bfloat162float(row[k]) - mean; + float v = __bfloat162float(row[k])-mean; vsum += v * v; } for (int off = 16; off > 0; off >>= 1) @@ -590,52 +590,52 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { - float v = (tid < (BLOCK + 31) / 32) ? smem[tid] : 0.0f; + float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; } __syncthreads(); - const float inv_std = rsqrtf(smem[0] / (float)K + eps); + const float inv_std = rsqrtf(smem[0]/(float)K+eps); for (int k = tid; k < K; k += BLOCK) { - float v = (__bfloat162float(row[k]) - mean) * inv_std; + float v = (__bfloat162float(row[k])-mean)*inv_std; float wv = __bfloat162float(w[k]); float bv = __bfloat162float(b[k]); - o[k] = __float2bfloat16(v * wv + bv); + o[k] = __float2bfloat16(v * wv+bv); } } __global__ void gelu_tanh_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* out, int N) { - const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); + const int i = (int)(blockIdx.x*blockDim.x+threadIdx.x); if (i >= N) return; float x = __bfloat162float(in[i]); const float kAlpha = 0.7978845608028654f; const float kBeta = 0.044715f; - float u = kAlpha * (x + kBeta * x * x * x); + float u = kAlpha * (x+kBeta * x * x * x); float t = tanhf(u); - out[i] = __float2bfloat16(0.5f * x * (1.0f + t)); + out[i] = __float2bfloat16(0.5f * x * (1.0f+t)); } __global__ void add_bias_bf16_kernel(const __nv_bfloat16* x, const __nv_bfloat16* bias, __nv_bfloat16* out, int M, int K) { const int m = (int)blockIdx.x; - const int k = (int)(blockIdx.y * blockDim.x + threadIdx.x); + const int k = (int)(blockIdx.y*blockDim.x+threadIdx.x); if (k >= K) return; - const size_t i = (size_t)m * K + k; - out[i] = __float2bfloat16(__bfloat162float(x[i]) + __bfloat162float(bias[k])); + const size_t i = (size_t)m * K+k; + out[i] = __float2bfloat16(__bfloat162float(x[i])+__bfloat162float(bias[k])); } __global__ void zero_tail_bf16_kernel(__nv_bfloat16* x, int total_cols, int start_col) { const int m = (int)blockIdx.x; - const int k = (int)(start_col + blockIdx.y * blockDim.x + threadIdx.x); + const int k = (int)(start_col+blockIdx.y*blockDim.x+threadIdx.x); if (k >= total_cols) return; - x[(size_t)m * total_cols + k] = __float2bfloat16(0.0f); + x[(size_t)m * total_cols+k] = __float2bfloat16(0.0f); } extern "C" void bitvla_layernorm_bf16(const __nv_bfloat16* x, const __nv_bfloat16* w, @@ -647,12 +647,12 @@ extern "C" void bitvla_layernorm_bf16(const __nv_bfloat16* x, const __nv_bfloat1 extern "C" void bitvla_gelu_tanh_bf16(const __nv_bfloat16* x, __nv_bfloat16* out, int N, cudaStream_t stream) { constexpr int B = 256; - gelu_tanh_bf16_kernel<<>>(x, out, N); + gelu_tanh_bf16_kernel<<>>(x, out, N); } extern "C" void bitvla_add_bias_bf16(const __nv_bfloat16* x, const __nv_bfloat16* bias, __nv_bfloat16* out, int M, int K, cudaStream_t stream) { constexpr int B = 256; - const int n_kb = (K + B - 1) / B; + const int n_kb = (K+B-1)/B; add_bias_bf16_kernel<<>>(x, bias, out, M, K); } extern "C" void bitvla_zero_tail_bf16(__nv_bfloat16* x, int M, int total_cols, @@ -660,8 +660,8 @@ extern "C" void bitvla_zero_tail_bf16(__nv_bfloat16* x, int M, int total_cols, if (start_col >= total_cols) return; constexpr int B = 128; - const int len = total_cols - start_col; - const int n_kb = (len + B - 1) / B; + const int len = total_cols-start_col; + const int n_kb = (len+B-1)/B; zero_tail_bf16_kernel<<>>(x, total_cols, start_col); } @@ -698,7 +698,7 @@ extern "C" int bitvla_lm_cuda_forward(bitvla_lm_cuda_ctx* ctx, } }; - CUDA_OK(cudaMemcpyAsync(ctx->d_h, d_in, (size_t)seq * ctx->hidden * sizeof(__nv_bfloat16), + CUDA_OK(cudaMemcpyAsync(ctx->d_h, d_in, (size_t)seq * ctx->hidden*sizeof(__nv_bfloat16), cudaMemcpyDeviceToDevice, stream)); if (dump_dir) { cudaStreamSynchronize(stream); diff --git a/src/kernels/bitvla/bitvla_lm_cuda.h b/src/kernels/bitvla/bitvla_lm_cuda.h index 60a64b3..ee79799 100644 --- a/src/kernels/bitvla/bitvla_lm_cuda.h +++ b/src/kernels/bitvla/bitvla_lm_cuda.h @@ -20,10 +20,10 @@ * packed format ("ladder int8xint2") alongside FP32 weight scales. This * header exposes: * - * * Standalone bf16 ops (norm / RoPE / softmax / activations) used by + * * Standalone bf16 ops (norm/RoPE/softmax/activations) used by * both the LM and the ViT. * * @ref bitvla_lm_cuda_ctx, an opaque context that owns the device-side - * LM state, plus its init / set-layer / forward / free entry points. + * LM state, plus its init/set-layer/forward/free entry points. * * All functions are @c extern @c "C" so they can be called from C++ or C * driver code. Streams are passed in explicitly; the kernels never call @@ -81,7 +81,7 @@ void bitvla_softmax_scaled_bf16(__nv_bfloat16* inout, float scale, int n_rows, int S, cudaStream_t stream); /** - * @brief Elementwise @c relu(g)^2 * u (BitVLA squared-ReLU FFN gate). + * @brief Elementwise @c relu(g)^2*u (BitVLA squared-ReLU FFN gate). * @param g Gate input (N), bf16 device pointer. * @param u Up input (N), bf16 device pointer. * @param out Output (N), bf16 device pointer. @@ -155,7 +155,7 @@ void bitvla_gather_rows_bf16(const __nv_bfloat16* in, __nv_bfloat16* out, cudaStream_t stream); /** - * @brief Affine LayerNorm in bf16 (mean/variance + scale + bias). + * @brief Affine LayerNorm in bf16 (mean/variance+scale+bias). * @param x Input matrix (M x K), bf16 device pointer. * @param w Per-channel scale (length K). * @param b Per-channel bias (length K). diff --git a/src/kernels/bitvla/bitvla_vit_cuda.cu b/src/kernels/bitvla/bitvla_vit_cuda.cu index 2185d28..7728639 100644 --- a/src/kernels/bitvla/bitvla_vit_cuda.cu +++ b/src/kernels/bitvla/bitvla_vit_cuda.cu @@ -31,16 +31,16 @@ extern "C" void bitvla_act_quant_cuda(const __nv_bfloat16* in, int8_t* out, int M, int K, cudaStream_t stream); __global__ void gelu_erf_bf16_kernel(const __nv_bfloat16* in, __nv_bfloat16* out, int N) { - const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); + const int i = (int)(blockIdx.x*blockDim.x+threadIdx.x); if (i >= N) return; const float x = __bfloat162float(in[i]); const float inv_sqrt2 = 0.7071067811865475f; - out[i] = __float2bfloat16(0.5f * x * (1.0f + erff(x * inv_sqrt2))); + out[i] = __float2bfloat16(0.5f * x * (1.0f+erff(x * inv_sqrt2))); } static void gelu_erf_bf16(const __nv_bfloat16* in, __nv_bfloat16* out, int N, cudaStream_t stream) { constexpr int B = 256; - gelu_erf_bf16_kernel<<>>(in, out, N); + gelu_erf_bf16_kernel<<>>(in, out, N); } #define CUDA_OK(call) do { cudaError_t e = (call); if (e != cudaSuccess) { \ @@ -97,14 +97,14 @@ bitvla_vit_cuda_ctx* bitvla_vit_cuda_init(int n_layers, int hidden, int n_heads, ctx->n_layers = n_layers; ctx->hidden = hidden; ctx->n_heads = n_heads; - ctx->head_dim = hidden / n_heads; + ctx->head_dim = hidden/n_heads; ctx->ffn = ffn; ctx->n_patches = n_patches; ctx->patch_flat = patch_flat; ctx->ln_eps = ln_eps; ctx->mm_out = mm_out; - ctx->ffn_pad = ((ffn + 127) / 128) * 128; + ctx->ffn_pad = ((ffn+127)/128)*128; ctx->layers.resize(n_layers); if (cublasCreate(&ctx->cublas) != CUBLAS_STATUS_SUCCESS) { @@ -120,17 +120,17 @@ bitvla_vit_cuda_ctx* bitvla_vit_cuda_init(int n_layers, int hidden, int n_heads, CUDA_OKV(cudaMalloc(&ctx->d_q_proj, (size_t) n_patches * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_k_proj, (size_t) n_patches * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_v_proj, (size_t) n_patches * hidden * bf16)); - CUDA_OKV(cudaMalloc(&ctx->d_q_HShd, (size_t) n_heads * n_patches * ctx->head_dim * bf16)); - CUDA_OKV(cudaMalloc(&ctx->d_k_HShd, (size_t) n_heads * n_patches * ctx->head_dim * bf16)); - CUDA_OKV(cudaMalloc(&ctx->d_v_HShd, (size_t) n_heads * n_patches * ctx->head_dim * bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_q_HShd, (size_t) n_heads * n_patches * ctx->head_dim*bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_k_HShd, (size_t) n_heads * n_patches * ctx->head_dim*bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_v_HShd, (size_t) n_heads * n_patches * ctx->head_dim*bf16)); CUDA_OKV(cudaMalloc(&ctx->d_scores, (size_t) n_heads * n_patches * n_patches * bf16)); - CUDA_OKV(cudaMalloc(&ctx->d_attn_out, (size_t) n_heads * n_patches * ctx->head_dim * bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_attn_out, (size_t) n_heads * n_patches * ctx->head_dim*bf16)); CUDA_OKV(cudaMalloc(&ctx->d_attn_merged, (size_t) n_patches * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_o_out, (size_t) n_patches * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_fc1_dense, (size_t) n_patches * ffn * bf16)); - CUDA_OKV(cudaMalloc(&ctx->d_fc1_padded, (size_t) n_patches * ctx->ffn_pad * bf16)); + CUDA_OKV(cudaMalloc(&ctx->d_fc1_padded, (size_t) n_patches * ctx->ffn_pad*bf16)); - CUDA_OKV(cudaMemset(ctx->d_fc1_padded, 0, (size_t) n_patches * ctx->ffn_pad * bf16)); + CUDA_OKV(cudaMemset(ctx->d_fc1_padded, 0, (size_t) n_patches * ctx->ffn_pad*bf16)); CUDA_OKV(cudaMalloc(&ctx->d_fc2_out, (size_t) n_patches * hidden * bf16)); CUDA_OKV(cudaMalloc(&ctx->d_mm_h1, (size_t) n_patches * mm_out * bf16)); return ctx; @@ -207,7 +207,7 @@ static int run_vit_layer(bitvla_vit_cuda_ctx* ctx, int L, cudaStream_t stream) { return -1; } - const float scl = 1.0f / std::sqrt((float) hd); + const float scl = 1.0f/std::sqrt((float) hd); bitvla_softmax_scaled_bf16(ctx->d_scores, scl, n_heads * seq, seq, stream); cbs = cublasGemmStridedBatchedEx( diff --git a/src/kernels/bitvla/bitvla_vit_cuda.h b/src/kernels/bitvla/bitvla_vit_cuda.h index 963ba5e..5fee0b6 100644 --- a/src/kernels/bitvla/bitvla_vit_cuda.h +++ b/src/kernels/bitvla/bitvla_vit_cuda.h @@ -14,7 +14,7 @@ /** * @file bitvla_vit_cuda.h - * @brief CUDA forward path for the BitVLA vision tower (ViT + mmproj). + * @brief CUDA forward path for the BitVLA vision tower (ViT+mmproj). * * Mirrors the layout of @ref bitvla_lm_cuda.h: an opaque * @ref bitvla_vit_cuda_ctx owns device buffers; per-layer weights are diff --git a/src/model.cpp b/src/model.cpp index cd3eab1..e2dd40e 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -35,7 +35,7 @@ namespace { bool ends_with_gguf(const std::string& p) { if (p.size() < 5) return false; - return std::strcmp(p.c_str() + p.size() - 5, ".gguf") == 0; + return std::strcmp(p.c_str()+p.size()-5, ".gguf") == 0; } bool detect_arch_gguf(const std::string& path, Arch* out) { diff --git a/src/model.h b/src/model.h index bf4372f..ce69199 100644 --- a/src/model.h +++ b/src/model.h @@ -152,7 +152,7 @@ struct Inputs { }; /** - * @brief Load a model from one (vision-baked) or two (mmproj + ckpt) GGUFs. + * @brief Load a model from one (vision-baked) or two (mmproj+ckpt) GGUFs. * * The architecture is detected from the checkpoint via * @ref detect_arch_from_ckpt. Fails loud: a missing file, unknown diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index d705acc..5107c90 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -69,23 +69,23 @@ struct LmLayerW { void bitvla_act_quant_op(ggml_tensor * dst, const ggml_tensor * a, int ith, int nth, void * ) { const int64_t cols = a->ne[0]; const int64_t rows = ggml_nrows(a); - const int64_t per = (rows + nth - 1) / nth; + const int64_t per = (rows+nth-1)/nth; const int64_t r0 = ith * per; - const int64_t r1 = std::min(rows, r0 + per); + const int64_t r1 = std::min(rows, r0+per); const float * src = (const float *) a->data; float * out = (float *) dst->data; for (int64_t r = r0; r < r1; ++r) { - const float * row_in = src + r * cols; - float * row_out = out + r * cols; + const float * row_in = src+r * cols; + float * row_out = out+r * cols; float amax = 0.0f; for (int64_t c = 0; c < cols; ++c) amax = std::max(amax, std::fabs(row_in[c])); if (amax < 1e-5f) amax = 1e-5f; - const float s = 127.0f / amax; - const float inv_s = 1.0f / s; + const float s = 127.0f/amax; + const float inv_s = 1.0f/s; for (int64_t c = 0; c < cols; ++c) { - float q = std::nearbyintf(row_in[c] * s); + float q = std::nearbyintf(row_in[c]*s); if (q > 127.0f) q = 127.0f; if (q < -128.0f) @@ -183,7 +183,7 @@ ggml_tensor * rmsnorm(ggml_context * C, ggml_tensor * x, ggml_tensor * w, float ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) head_dim); + const float scale = 1.0f/std::sqrt((float) head_dim); ggml_tensor * x1 = layernorm(C, x, w.ln1w, w.ln1b, ln_eps); ggml_tensor * q = bit_linear(C, w.Wq, w.bq, x1); ggml_tensor * k = bit_linear(C, w.Wk, w.bk, x1); @@ -205,7 +205,7 @@ ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml_tensor ggml_tensor * build_lm_layer(ggml_context * C, const BitvlaModelArch & m, const LmLayerW & w, ggml_tensor * h, ggml_tensor * positions, int64_t seq) { const int64_t hd = m.lm_head_dim, n_q = m.lm_q, n_kv = m.lm_kv, hq = n_q * hd; - const float scale = 1.0f / std::sqrt((float) hd); + const float scale = 1.0f/std::sqrt((float) hd); ggml_tensor * hn = rmsnorm(C, h, w.attn_norm, m.lm_rms_eps); ggml_tensor * qp = bit_linear(C, w.Wq, nullptr, hn); @@ -242,11 +242,11 @@ bool parse_stats_json(const std::string & js, const char * env_key, std::vector & q01, std::vector & q99, std::vector & mask, std::string & resolved_key) { auto find_obj = [&](const std::string & where, const std::string & key) -> std::string { - const std::string q = std::string("\"") + key + "\""; + const std::string q = std::string("\"")+key + "\""; size_t p = where.find(q); if (p == std::string::npos) return ""; - p = where.find(':', p + q.size()); if (p == std::string::npos) return ""; + p = where.find(':', p+q.size()); if (p == std::string::npos) return ""; size_t s = where.find('{', p); if (s == std::string::npos) return ""; - int depth = 1; size_t i = s + 1; + int depth = 1; size_t i = s+1; while (i < where.size() && depth > 0) { if (where[i] == '{') depth++; else if (where[i] == '}') depth--; @@ -254,14 +254,14 @@ bool parse_stats_json(const std::string & js, const char * env_key, break; i++; } - return where.substr(s, i - s + 1); + return where.substr(s, i-s+1); }; auto parse_array_floats = [](const std::string & body, const char * key, std::vector & out) { - const std::string q = std::string("\"") + key + "\""; + const std::string q = std::string("\"")+key + "\""; size_t p = body.find(q); if (p == std::string::npos) return false; p = body.find('[', p); if (p == std::string::npos) return false; size_t e = body.find(']', p); if (e == std::string::npos) return false; - std::string inner = body.substr(p + 1, e - p - 1); + std::string inner = body.substr(p+1, e-p-1); out.clear(); const char * s = inner.c_str(); while (*s) { @@ -279,11 +279,11 @@ bool parse_stats_json(const std::string & js, const char * env_key, return true; }; auto parse_array_bools = [](const std::string & body, const char * key, std::vector & out) { - const std::string q = std::string("\"") + key + "\""; + const std::string q = std::string("\"")+key + "\""; size_t p = body.find(q); if (p == std::string::npos) return false; p = body.find('[', p); if (p == std::string::npos) return false; size_t e = body.find(']', p); if (e == std::string::npos) return false; - std::string inner = body.substr(p + 1, e - p - 1); + std::string inner = body.substr(p+1, e-p-1); out.clear(); size_t i = 0; while (i < inner.size()) { @@ -313,10 +313,10 @@ bool parse_stats_json(const std::string & js, const char * env_key, size_t p = js.find('"'); if (p == std::string::npos) return false; - size_t q = js.find('"', p + 1); + size_t q = js.find('"', p+1); if (q == std::string::npos) return false; - suite = js.substr(p + 1, q - p - 1); + suite = js.substr(p+1, q-p-1); } resolved_key = suite; @@ -374,15 +374,15 @@ bool load_config(const gguf_reader & g, BitvlaModelArch & m, Config & cfg) { // predict() sizes the patch buffer from n_patches but fills it by walking the // image grid, so a KV that disagrees with the geometry overruns the buffer. - if (m.patch_size <= 0 || m.image_size <= 0 || m.image_size % m.patch_size != 0 || - m.n_patches != (m.image_size / m.patch_size) * (m.image_size / m.patch_size)) { + if (m.patch_size <= 0 || m.image_size <= 0 || m.image_size%m.patch_size != 0 || + m.n_patches != (m.image_size/m.patch_size)*(m.image_size/m.patch_size)) { std::fprintf(stderr, "vla(bitvla): n_patches %lld does not match image %lld / patch %lld\n", (long long) m.n_patches, (long long) m.image_size, (long long) m.patch_size); return false; } // The CUDA LM writes seq*q_heads*head_dim into buffers sized seq*hidden. - if (m.lm_kv <= 0 || m.lm_head_dim <= 0 || m.lm_q % m.lm_kv != 0 || - m.lm_q * m.lm_head_dim != m.lm_hidden) { + if (m.lm_kv <= 0 || m.lm_head_dim <= 0 || m.lm_q%m.lm_kv != 0 || + m.lm_q*m.lm_head_dim != m.lm_hidden) { std::fprintf(stderr, "vla(bitvla): lm q_heads %lld x head_dim %lld does not match hidden %lld\n", (long long) m.lm_q, (long long) m.lm_head_dim, (long long) m.lm_hidden); return false; @@ -434,14 +434,14 @@ static void recover_ternary_and_scale(const float* W, int64_t n, double s = 0.0; for (int64_t i = 0; i < n; ++i) s += std::fabs((double) W[i]); - float mean = n > 0 ? (float) (s / (double) n) : 0.0f; + float mean = n > 0 ? (float) (s/(double) n) : 0.0f; if (mean < 1e-5f) mean = 1e-5f; absmean = mean; - const float inv = 1.0f / mean; + const float inv = 1.0f/mean; ternary.resize(n); for (int64_t i = 0; i < n; ++i) { - float q = std::nearbyintf(W[i] * inv); + float q = std::nearbyintf(W[i]*inv); if (q > 1.0f) q = 1.0f; if (q < -1.0f) @@ -453,30 +453,30 @@ static void recover_ternary_and_scale(const float* W, int64_t n, static std::vector pack_ladder_int2(const int8_t* W, int64_t N, int64_t K) { constexpr int N_BLOCK = 16, K_BLOCK = 8, K_PER_LOOP = 16; constexpr int WMMA_K = 32, K_PER_ITER = K_PER_LOOP * K_BLOCK; - const int64_t n_slots = N * K / 16; - std::vector out(N * K / 4, 0); + const int64_t n_slots = N * K/16; + std::vector out(N * K/4, 0); for (int64_t s = 0; s < n_slots; ++s) { - const int64_t slots_per_block = (N_BLOCK * K) / 16; - const int64_t n_block = s / slots_per_block; - const int64_t in_block = s % slots_per_block; - const int64_t k_0 = in_block / 128; - const int64_t in_k0 = in_block % 128; - const int64_t major_k = in_k0 / 32; - const int64_t in_major = in_k0 % 32; - const int64_t y_half = in_major / 16; - const int64_t in_yhalf = in_major % 16; - const int64_t sub_k = in_yhalf / 8; - const int64_t y_in_h = in_yhalf % 8; - const int64_t n_global = n_block * N_BLOCK + y_half * 8 + y_in_h; - const int64_t k_sub = k_0 * K_PER_ITER + major_k * WMMA_K + sub_k * K_PER_LOOP; + const int64_t slots_per_block = (N_BLOCK * K)/16; + const int64_t n_block = s/slots_per_block; + const int64_t in_block = s%slots_per_block; + const int64_t k_0 = in_block/128; + const int64_t in_k0 = in_block%128; + const int64_t major_k = in_k0/32; + const int64_t in_major = in_k0%32; + const int64_t y_half = in_major/16; + const int64_t in_yhalf = in_major%16; + const int64_t sub_k = in_yhalf/8; + const int64_t y_in_h = in_yhalf%8; + const int64_t n_global = n_block * N_BLOCK+y_half*8+y_in_h; + const int64_t k_sub = k_0*K_PER_ITER+major_k * WMMA_K+sub_k * K_PER_LOOP; for (int byte_i = 0; byte_i < 4; ++byte_i) { uint8_t b = 0; for (int j = 0; j < 4; ++j) { - const int t = (int) W[n_global * K + (k_sub + byte_i + 4 * j)]; - const uint8_t enc = (uint8_t)(t + 2) & 0x3; - b |= (enc << (2 * j)); + const int t = (int) W[n_global * K+(k_sub+byte_i+4*j)]; + const uint8_t enc = (uint8_t)(t+2) & 0x3; + b |= (enc << (2*j)); } - out[s * 4 + byte_i] = b; + out[s*4+byte_i] = b; } } return out; @@ -533,8 +533,8 @@ static int8_t* pack_and_upload_fused(const std::vector& wptrs, for (size_t i = 0; i < wptrs.size(); ++i) { std::vector tern; float sc; - recover_ternary_and_scale(wptrs[i], Ns[i] * K, tern, sc); - std::memcpy(stacked.data() + row_off * K, tern.data(), (size_t) Ns[i] * K); + recover_ternary_and_scale(wptrs[i], Ns[i]*K, tern, sc); + std::memcpy(stacked.data()+row_off * K, tern.data(), (size_t) Ns[i]*K); out_scales.push_back(sc); row_off += Ns[i]; } @@ -629,7 +629,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, ggml_backend_cpu_set_n_threads(m->backend, m->n_threads); std::printf("vla(bitvla): ggml backend = CPU (%d threads) - CUDA LM module activates below if available\n", m->n_threads); - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(bitvla): ggml_init(ctx_weights) failed\n"); @@ -712,7 +712,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, return nullptr; std::printf("vla(bitvla): weights resident in %.2f GiB (%s); image_id=%d proprio_id=%d action_begin_id=%d stop_id=%d\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), m->packed_int2 ? "int2-packed + F32 sidecars" : (m->matmul_type == GGML_TYPE_F32 ? "F32" : "BF16"), m->image_token_id, m->proprio_pad_id, m->action_begin_id, m->stop_id); @@ -740,7 +740,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, if (m->packed_int2) { int8_t * dp = upload_int8((const uint8_t*) t->data, ggml_nbytes(t), m->cuda_devptrs); std::string nm = ggml_get_name(t); - std::string sn = nm.substr(0, nm.size() - 7) + ".scale"; + std::string sn = nm.substr(0, nm.size()-7) + ".scale"; std::vector sc = g.read_f32(sn.c_str()); if (sc.empty()) { std::fprintf(stderr, "vla(bitvla): int2 tensor %s has no %s sidecar\n", @@ -771,7 +771,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, lyr.ffn_sub_norm_w = upload_bf16_from_f32((const float*) m->lm[L].ffn_sub_norm->data, m->lm_inter, m->cuda_devptrs); const int64_t hq_dim = m->lm_q * m->lm_head_dim; - const int64_t hkv_dim = m->lm_kv * m->lm_head_dim; + const int64_t hkv_dim = m->lm_kv*m->lm_head_dim; { auto r = load_bit(m->lm[L].Wq, hq_dim, m->lm_hidden); lyr.q_packed = r.first; @@ -827,13 +827,13 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, __nv_bfloat16* onorm = upload_bf16_from_f32((const float*) m->lm_output_norm->data, m->lm_hidden, m->cuda_devptrs); bitvla_lm_cuda_set_output_norm(m->lm_cuda_ctx, onorm); - cudaError_t lm_ce = cudaMalloc(&m->d_inputs_embeds, (size_t) max_seq * m->lm_hidden * sizeof(__nv_bfloat16)); + cudaError_t lm_ce = cudaMalloc(&m->d_inputs_embeds, (size_t) max_seq * m->lm_hidden*sizeof(__nv_bfloat16)); if (lm_ce == cudaSuccess) - lm_ce = cudaMalloc(&m->d_last_hidden, (size_t) max_seq * m->lm_hidden * sizeof(__nv_bfloat16)); + lm_ce = cudaMalloc(&m->d_last_hidden, (size_t) max_seq * m->lm_hidden*sizeof(__nv_bfloat16)); if (lm_ce == cudaSuccess) - lm_ce = cudaMalloc(&m->d_action_hidden, (size_t) (m->num_actions_chunk * m->action_dim) * m->lm_hidden * sizeof(__nv_bfloat16)); + lm_ce = cudaMalloc(&m->d_action_hidden, (size_t) (m->num_actions_chunk*m->action_dim)*m->lm_hidden*sizeof(__nv_bfloat16)); if (lm_ce == cudaSuccess) - lm_ce = cudaMalloc(&m->d_action_ids, (size_t) (m->num_actions_chunk * m->action_dim) * sizeof(int32_t)); + lm_ce = cudaMalloc(&m->d_action_ids, (size_t) (m->num_actions_chunk*m->action_dim)*sizeof(int32_t)); // only enable the CUDA LM once every work buffer is really allocated. if (lm_ce != cudaSuccess) { std::fprintf(stderr, "vla(bitvla): CUDA LM buffer alloc failed (%s); using CPU LM\n", @@ -844,18 +844,18 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->d_action_hidden = nullptr; m->d_action_ids = nullptr; } else { m->cuda_lm_ready = true; - const size_t packed_bytes = (size_t) m->lm_layers * ( - (size_t)(m->lm_q + 2*m->lm_kv) * m->lm_head_dim * m->lm_hidden / 4 + - (size_t) m->lm_hidden * m->lm_hidden / 4 + - (size_t) 2 * m->lm_inter * m->lm_hidden / 4 + - (size_t) m->lm_hidden * m->lm_inter / 4); + const size_t packed_bytes = (size_t) m->lm_layers*( + (size_t)(m->lm_q+2*m->lm_kv)*m->lm_head_dim*m->lm_hidden/4 + + (size_t) m->lm_hidden*m->lm_hidden/4 + + (size_t) 2*m->lm_inter*m->lm_hidden/4 + + (size_t) m->lm_hidden*m->lm_inter/4); std::printf("vla(bitvla): CUDA LM forward ENABLED - packed int2 weights = %.2f MiB, max_seq=%d\n", - packed_bytes / (1024.0 * 1024.0), max_seq); + packed_bytes/(1024.0*1024.0), max_seq); } - const int patch_flat = 3 * m->patch_size * m->patch_size; + const int patch_flat = 3*m->patch_size*m->patch_size; const int mm_out = (int) m->lm_hidden; - const int ffn_pad = ((m->vit_inter + 127) / 128) * 128; + const int ffn_pad = ((m->vit_inter+127)/128)*128; m->vit_cuda_ctx = bitvla_vit_cuda_init( (int) m->vit_layers, (int) m->vit_hidden, (int) m->vit_heads, (int) m->vit_inter, (int) m->n_patches, patch_flat, @@ -913,12 +913,12 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, const float* W = (const float*) m->vit[L].Wfc2->data; std::vector tern; float scale; - recover_ternary_and_scale(W, m->vit_hidden * m->vit_inter, tern, scale); + recover_ternary_and_scale(W, m->vit_hidden*m->vit_inter, tern, scale); - std::vector padded((size_t) m->vit_hidden * ffn_pad, 0); + std::vector padded((size_t) m->vit_hidden*ffn_pad, 0); for (int64_t n = 0; n < m->vit_hidden; ++n) { - std::memcpy(padded.data() + n * ffn_pad, - tern.data() + n * m->vit_inter, + std::memcpy(padded.data()+n * ffn_pad, + tern.data()+n * m->vit_inter, (size_t) m->vit_inter); } @@ -931,9 +931,9 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, } if (vit_ok) { - __nv_bfloat16* pe_w = upload_bf16_from_f32((const float*) m->vit_patch_w->data, m->vit_hidden * patch_flat, m->cuda_devptrs); + __nv_bfloat16* pe_w = upload_bf16_from_f32((const float*) m->vit_patch_w->data, m->vit_hidden*patch_flat, m->cuda_devptrs); __nv_bfloat16* pe_b = upload_bf16_from_f32((const float*) m->vit_patch_b->data, m->vit_hidden, m->cuda_devptrs); - __nv_bfloat16* pos_e = upload_bf16_from_f32((const float*) m->vit_pos->data, m->n_patches * m->vit_hidden, m->cuda_devptrs); + __nv_bfloat16* pos_e = upload_bf16_from_f32((const float*) m->vit_pos->data, m->n_patches*m->vit_hidden, m->cuda_devptrs); bitvla_vit_cuda_set_embed(m->vit_cuda_ctx, pe_w, pe_b, pos_e); __nv_bfloat16* mm_W1 = upload_bf16_from_f32((const float*) m->mm_l1_w->data, mm_out * m->vit_hidden, m->cuda_devptrs); @@ -942,15 +942,15 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, __nv_bfloat16* mm_b2 = upload_bf16_from_f32((const float*) m->mm_l2_b->data, mm_out, m->cuda_devptrs); bitvla_vit_cuda_set_mmproj(m->vit_cuda_ctx, mm_W1, mm_b1, mm_W2, mm_b2); - cudaMalloc(&m->d_vit_patches, (size_t) m->n_patches * patch_flat * sizeof(__nv_bfloat16)); - cudaMalloc(&m->d_vit_img_embeds, (size_t) m->n_patches * mm_out * sizeof(__nv_bfloat16)); + cudaMalloc(&m->d_vit_patches, (size_t) m->n_patches*patch_flat * sizeof(__nv_bfloat16)); + cudaMalloc(&m->d_vit_img_embeds, (size_t) m->n_patches*mm_out * sizeof(__nv_bfloat16)); m->cuda_vit_ready = true; - const size_t vit_packed_bytes = (size_t) m->vit_layers * ( - 4 * (size_t) m->vit_hidden * m->vit_hidden / 4 + - (size_t) m->vit_inter * m->vit_hidden / 4 + - (size_t) m->vit_hidden * ffn_pad / 4); + const size_t vit_packed_bytes = (size_t) m->vit_layers*( + 4*(size_t) m->vit_hidden*m->vit_hidden/4 + + (size_t) m->vit_inter*m->vit_hidden/4 + + (size_t) m->vit_hidden*ffn_pad/4); std::printf("vla(bitvla): CUDA ViT forward ENABLED - packed int2 weights = %.2f MiB, ffn_pad=%d\n", - vit_packed_bytes / (1024.0 * 1024.0), ffn_pad); + vit_packed_bytes/(1024.0*1024.0), ffn_pad); } else { bitvla_vit_cuda_free(m->vit_cuda_ctx); m->vit_cuda_ctx = nullptr; @@ -1000,12 +1000,12 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, const int64_t h = m->lm_hidden, p = m->proprio_dim, a = m->action_dim; const int64_t adh = a * h; const size_t bytes = (size_t)( - h*p + h + h*h + h + - 2*adh + h*adh + h + - 2 * (2*h + h*h + h) + - 2*h + a*h + a) * sizeof(float); + h*p+h+h*h+h + + 2*adh+h*adh+h + + 2*(2*h+h*h+h) + + 2*h+a*h+a)*sizeof(float); std::printf("vla(bitvla): CUDA fp32head (ProprioProj+ActionHead) ENABLED - weights = %.2f MiB on GPU\n", - bytes / (1024.0 * 1024.0)); + bytes/(1024.0*1024.0)); } else { std::fprintf(stderr, "vla(bitvla): bitvla_fp32head_cuda_init failed; falling back to CPU ggml head\n"); } @@ -1038,7 +1038,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, bytes_kept += nb; } } - const double before_gb = ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0); + const double before_gb = ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0); ggml_backend_buffer_free(m->weight_buf); m->weight_buf = nullptr; @@ -1051,7 +1051,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, } else { std::printf("vla(bitvla): freed weight_buf (%.2f GiB); kept %.2f MiB of CPU-resident " "ProprioProj+ActionHead weights as standalone copies\n", - before_gb, bytes_kept / (1024.0 * 1024.0)); + before_gb, bytes_kept/(1024.0*1024.0)); } } #endif @@ -1095,7 +1095,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { const int64_t H = image_size, P = patch_size, N = n_patches; const int64_t hidden_v = vit_hidden, hidden_l = lm_hidden; - const int64_t patch_flat = 3 * P * P; + const int64_t patch_flat = 3*P * P; const bool use_precomp_img = (in.precomputed_img_emb != nullptr) && in.n_img_views > 0; const int64_t n_views = use_precomp_img ? (int64_t) in.n_img_views : (int64_t) in.n_images; @@ -1119,24 +1119,24 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { return {}; } - for (int64_t pi = 0; pi < H / P; ++pi) - for (int64_t pj = 0; pj < H / P; ++pj) { - const int64_t p_idx = pi * (H / P) + pj; - float * p_dst = patches.data() + p_idx * patch_flat; + for (int64_t pi = 0; pi < H/P; ++pi) + for (int64_t pj = 0; pj < H/P; ++pj) { + const int64_t p_idx = pi * (H/P)+pj; + float * p_dst = patches.data()+p_idx * patch_flat; int64_t k = 0; for (int64_t c = 0; c < 3; ++c) for (int64_t kh = 0; kh < P; ++kh) for (int64_t kw = 0; kw < P; ++kw) { - const int64_t h = pi * P + kh; - const int64_t w = pj * P + kw; + const int64_t h = pi * P+kh; + const int64_t w = pj * P+kw; float px; if (iv.format == PixelFormat::U8) { const uint8_t * p = (const uint8_t *) iv.data; - px = (float) p[h * H * 3 + w * 3 + c] / 127.5f - 1.0f; + px = (float) p[h * H*3+w*3+c]/127.5f-1.0f; } else { const float * p = (const float *) iv.data; - px = p[h * H * 3 + w * 3 + c] * 2.0f - 1.0f; + px = p[h * H*3+w*3+c]*2.0f-1.0f; } p_dst[k++] = px; } @@ -1148,12 +1148,12 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::vector patches_bf16((size_t) N * patch_flat); for (size_t i = 0; i < patches_bf16.size(); ++i) patches_bf16[i] = f32_to_bf16_u16(patches[i]); - cudaMemcpy(d_vit_patches, patches_bf16.data(), patches_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice); + cudaMemcpy(d_vit_patches, patches_bf16.data(), patches_bf16.size()*sizeof(uint16_t), cudaMemcpyHostToDevice); int rc = bitvla_vit_cuda_forward(vit_cuda_ctx, d_vit_patches, d_vit_img_embeds, 0); if (rc != 0) { std::fprintf(stderr, "vla(bitvla): CUDA ViT forward failed (view %lld)\n", (long long) v); return {}; } std::vector img_bf16((size_t) N * hidden_l); - cudaMemcpy(img_bf16.data(), d_vit_img_embeds, img_bf16.size() * sizeof(uint16_t), cudaMemcpyDeviceToHost); - float* dst = img_embeds_host.data() + (size_t) v * N * hidden_l; + cudaMemcpy(img_bf16.data(), d_vit_img_embeds, img_bf16.size()*sizeof(uint16_t), cudaMemcpyDeviceToHost); + float* dst = img_embeds_host.data()+(size_t) v * N * hidden_l; for (size_t i = 0; i < img_bf16.size(); ++i) { uint32_t u = ((uint32_t) img_bf16[i]) << 16; float f; std::memcpy(&f, &u, 4); @@ -1163,7 +1163,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { #endif { - ggml_context * ctx = vision_scratch.reset((size_t) 24 * 1024 * 1024); + ggml_context * ctx = vision_scratch.reset((size_t) 24*1024*1024); ggml_tensor * x_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, patch_flat, N); ggml_set_name(x_in, "patches"); @@ -1188,16 +1188,16 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(bitvla): vision graph compute failed (view %lld)\n", (long long) v); return {}; } - ggml_backend_tensor_get(mm2, img_embeds_host.data() + (size_t) v * N * hidden_l, 0, (size_t) N * hidden_l * sizeof(float)); + ggml_backend_tensor_get(mm2, img_embeds_host.data()+(size_t) v * N * hidden_l, 0, (size_t) N * hidden_l * sizeof(float)); } } - stats.ms_vision = std::chrono::duration(clk::now() - t_v0).count(); + stats.ms_vision = std::chrono::duration(clk::now()-t_v0).count(); } else { - img_embeds_host.assign(in.precomputed_img_emb, in.precomputed_img_emb + (size_t) n_views * N * hidden_l); + img_embeds_host.assign(in.precomputed_img_emb, in.precomputed_img_emb+(size_t) n_views * N * hidden_l); } _dump_bin("mm_proj_out", img_embeds_host.data(), img_embeds_host.size()); - _dump_manifest(std::string("mm_proj_out fp32 ") + std::to_string(n_views) + " " + std::to_string(N) + " " + std::to_string(hidden_l)); + _dump_manifest(std::string("mm_proj_out fp32 ")+std::to_string(n_views) + " " + std::to_string(N) + " " + std::to_string(hidden_l)); std::vector proprio_embed_host((size_t) hidden_l); // Like the other archs: a caller may leave the proprio vector out. @@ -1212,7 +1212,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } else #endif { - ggml_context * ctx = proprio_scratch.reset((size_t) 4 * 1024 * 1024); + ggml_context * ctx = proprio_scratch.reset((size_t) 4*1024*1024); ggml_tensor * x_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, proprio_dim, 1); ggml_set_name(x_in, "state"); ggml_tensor * h1 = ggml_add(ctx, ggml_mul_mat(ctx, pp_fc1_w, x_in), pp_fc1_b); @@ -1229,7 +1229,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } _dump_bin("proprio_features", proprio_embed_host.data(), proprio_embed_host.size()); - _dump_manifest(std::string("proprio_features fp32 1 ") + std::to_string(hidden_l)); + _dump_manifest(std::string("proprio_features fp32 1 ")+std::to_string(hidden_l)); const int64_t n_lang_in = (int64_t) in.n_lang; const int64_t n_img_tok = n_views * N; @@ -1259,8 +1259,8 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } const int64_t seq = full_prefix - ? (n_lang_in + n_action + 1) - : (n_lang_in + n_img_tok + 1 + n_action + 1); + ? (n_lang_in+n_action+1) + : (n_lang_in+n_img_tok+1+n_action+1); if (seq > lm_max_pos) { std::fprintf(stderr, "vla(bitvla): seq=%lld > lm_max_pos=%lld\n", (long long) seq, (long long) lm_max_pos); return {}; @@ -1270,7 +1270,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { if (full_prefix) { - std::vector ids(in.lang_tokens, in.lang_tokens + n_lang_in); + std::vector ids(in.lang_tokens, in.lang_tokens+n_lang_in); for (int32_t id : ids) { if (id < 0 || id >= vocab_size) { std::fprintf(stderr, "vla(bitvla): prompt token %d out of vocab\n", id); return {}; @@ -1282,24 +1282,24 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < n_lang_in; ++i) { const int32_t tok = in.lang_tokens[i]; if (tok == image_token_id) { - std::memcpy(inputs_embeds.data() + (size_t) i * hidden_l, - img_embeds_host.data() + (size_t) k_img * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) i * hidden_l, + img_embeds_host.data()+(size_t) k_img * hidden_l, (size_t) hidden_l * sizeof(float)); k_img++; } else if (tok == proprio_pad_id) { - std::memcpy(inputs_embeds.data() + (size_t) i * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) i * hidden_l, proprio_embed_host.data(), (size_t) hidden_l * sizeof(float)); } } - std::memcpy(inputs_embeds.data() + (size_t) (seq - 1) * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) (seq-1)*hidden_l, stop_embed.data(), (size_t) hidden_l * sizeof(float)); } else { const int64_t n_prompt = n_lang_in; if (n_prompt > 0) { - std::vector ids(in.lang_tokens, in.lang_tokens + n_prompt); + std::vector ids(in.lang_tokens, in.lang_tokens+n_prompt); for (int32_t id : ids) { if (id < 0 || id >= vocab_size) { std::fprintf(stderr, "vla(bitvla): prompt token %d out of vocab\n", id); return {}; @@ -1307,16 +1307,16 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } if (!emb_reader.fetch_rows_f32("token_embd.weight", ids, inputs_embeds.data(), hidden_l)) return {}; } - std::memcpy(inputs_embeds.data() + (size_t) n_prompt * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) n_prompt * hidden_l, img_embeds_host.data(), (size_t) n_img_tok * hidden_l * sizeof(float)); - std::memcpy(inputs_embeds.data() + (size_t) (n_prompt + n_img_tok) * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) (n_prompt+n_img_tok)*hidden_l, proprio_embed_host.data(), (size_t) hidden_l * sizeof(float)); - std::memcpy(inputs_embeds.data() + (size_t) (seq - 1) * hidden_l, + std::memcpy(inputs_embeds.data()+(size_t) (seq-1)*hidden_l, stop_embed.data(), (size_t) hidden_l * sizeof(float)); } _dump_bin("inputs_embeds", inputs_embeds.data(), inputs_embeds.size()); - _dump_manifest(std::string("inputs_embeds fp32 1 ") + std::to_string(seq) + + _dump_manifest(std::string("inputs_embeds fp32 1 ")+std::to_string(seq) + " " + std::to_string(hidden_l)); std::vector last_hidden_at_actions((size_t) n_action * hidden_l); @@ -1328,19 +1328,19 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::vector in_bf16((size_t) seq * hidden_l); for (size_t i = 0; i < in_bf16.size(); ++i) in_bf16[i] = f32_to_bf16_u16(inputs_embeds[i]); - cudaMemcpy(d_inputs_embeds, in_bf16.data(), in_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice); + cudaMemcpy(d_inputs_embeds, in_bf16.data(), in_bf16.size()*sizeof(uint16_t), cudaMemcpyHostToDevice); int rc = bitvla_lm_cuda_forward(lm_cuda_ctx, d_inputs_embeds, d_last_hidden, (int) seq, 0); if (rc != 0) { std::fprintf(stderr, "vla(bitvla): CUDA LM forward failed\n"); return {}; } std::vector aids(n_action); for (int64_t i = 0; i < n_action; ++i) - aids[i] = (int32_t) (seq - 2 - n_action + i); + aids[i] = (int32_t) (seq-2-n_action+i); cudaMemcpy(d_action_ids, aids.data(), n_action * sizeof(int32_t), cudaMemcpyHostToDevice); bitvla_gather_rows_bf16(d_last_hidden, d_action_hidden, d_action_ids, (int) n_action, (int) hidden_l, 0); std::vector out_bf16((size_t) n_action * hidden_l); - cudaMemcpy(out_bf16.data(), d_action_hidden, out_bf16.size() * sizeof(uint16_t), cudaMemcpyDeviceToHost); + cudaMemcpy(out_bf16.data(), d_action_hidden, out_bf16.size()*sizeof(uint16_t), cudaMemcpyDeviceToHost); for (size_t i = 0; i < out_bf16.size(); ++i) { uint32_t u = ((uint32_t) out_bf16[i]) << 16; float f; std::memcpy(&f, &u, 4); @@ -1349,7 +1349,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } else #endif { - ggml_context * ctx = lm_scratch.reset((size_t) 64 * 1024 * 1024); + ggml_context * ctx = lm_scratch.reset((size_t) 64*1024*1024); ggml_tensor * x_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_l, seq); ggml_set_name(x_in, "inputs_embeds"); ggml_tensor * positions = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, seq); @@ -1378,17 +1378,17 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(positions, pos_v.data(), 0, ggml_nbytes(positions)); std::vector aids(n_action); for (int64_t i = 0; i < n_action; ++i) - aids[i] = (int32_t) (seq - 2 - n_action + i); + aids[i] = (int32_t) (seq-2-n_action+i); ggml_backend_tensor_set(action_ids, aids.data(), 0, ggml_nbytes(action_ids)); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): lm prefill compute failed\n"); return {}; } ggml_backend_tensor_get(action_hidden, last_hidden_at_actions.data(), 0, (size_t) n_action * hidden_l * sizeof(float)); } if (timing_phase) - stats.ms_prefill = std::chrono::duration(clk::now() - t_p0).count(); + stats.ms_prefill = std::chrono::duration(clk::now()-t_p0).count(); _dump_bin("ah_input", last_hidden_at_actions.data(), last_hidden_at_actions.size()); - _dump_manifest(std::string("ah_input fp32 1 ") + std::to_string(num_actions_chunk) + + _dump_manifest(std::string("ah_input fp32 1 ")+std::to_string(num_actions_chunk) + " " + std::to_string(action_dim * hidden_l)); const int64_t chunk = num_actions_chunk; @@ -1405,7 +1405,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { } else #endif { - ggml_context * ctx = head_scratch.reset((size_t) 8 * 1024 * 1024); + ggml_context * ctx = head_scratch.reset((size_t) 8*1024*1024); ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, in_dim, chunk); ggml_set_name(x, "x"); @@ -1434,27 +1434,27 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_backend_tensor_get(y, normalized_actions.data(), 0, (size_t) chunk * action_dim * sizeof(float)); } if (timing_phase) - stats.ms_denoise = std::chrono::duration(clk::now() - t_d0).count(); + stats.ms_denoise = std::chrono::duration(clk::now()-t_d0).count(); _dump_bin("ah_norm_actions", normalized_actions.data(), normalized_actions.size()); - _dump_manifest(std::string("ah_norm_actions fp32 1 ") + std::to_string(num_actions_chunk) + + _dump_manifest(std::string("ah_norm_actions fp32 1 ")+std::to_string(num_actions_chunk) + " " + std::to_string(action_dim)); std::vector actions = std::move(normalized_actions); for (int64_t t = 0; t < chunk; ++t) { for (int64_t d = 0; d < action_dim; ++d) { - const float a = actions[t * action_dim + d]; + const float a = actions[t * action_dim+d]; if (unnorm_mask[d]) { - actions[t * action_dim + d] = 0.5f * (a + 1.0f) * (q99[d] - q01[d] + 1e-8f) + q01[d]; + actions[t * action_dim+d] = 0.5f * (a+1.0f)*(q99[d]-q01[d]+1e-8f)+q01[d]; } } } stats.ms_inference = (timing_phase - ? stats.ms_prefill + stats.ms_denoise - : std::chrono::duration(clk::now() - t_start).count() - stats.ms_vision); - stats.ms_total = std::chrono::duration(clk::now() - t_start).count(); + ? stats.ms_prefill+stats.ms_denoise + : std::chrono::duration(clk::now()-t_start).count()-stats.ms_vision); + stats.ms_total = std::chrono::duration(clk::now()-t_start).count(); return actions; } diff --git a/src/models/dit_common.h b/src/models/dit_common.h index c293180..a17cf40 100644 --- a/src/models/dit_common.h +++ b/src/models/dit_common.h @@ -49,34 +49,34 @@ inline void timesteps_proj(int64_t bucket, std::vector & out) { const int64_t half = 128; const float lm = std::log(10000.0f); const float t = (float) bucket; out.assign(256, 0.0f); for (int64_t i = 0; i < half; ++i) { - const float emb = t * std::exp(-lm * (float) i / (float) (half - 1)); + const float emb = t * std::exp(-lm * (float) i/(float) (half-1)); out[i] = std::cos(emb); - out[half + i] = std::sin(emb); + out[half+i] = std::sin(emb); } } // Broadcast across the horizon. sin first, then cos. inline void action_sinusoid(int64_t bucket, int64_t dim, int64_t T, std::vector & out) { - const int64_t half = dim / 2; const float step = std::log(10000.0f) / (float) half; const float t = (float) bucket; + const int64_t half = dim/2; const float step = std::log(10000.0f)/(float) half; const float t = (float) bucket; out.assign((size_t) T * dim, 0.0f); for (int64_t tk = 0; tk < T; ++tk) for (int64_t i = 0; i < half; ++i) { const float emb = t * std::exp(-(float) i * step); - out[tk * dim + i] = std::sin(emb); - out[tk * dim + half + i] = std::cos(emb); + out[tk * dim+i] = std::sin(emb); + out[tk * dim+half+i] = std::cos(emb); } } // Log-spaced periods rather than frequencies, the openpi convention shared by // pi0, pi0.5 and SmolVLA. inline std::vector sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { - const int64_t half = dim / 2; + const int64_t half = dim/2; std::vector out(dim); for (int64_t i = 0; i < half; ++i) { - const double frac = (half == 1) ? 0.0 : double(i) / double(half - 1); - const double period = min_p * std::pow(max_p / min_p, frac); - const double s = (2.0 * M_PI / period) * t; + const double frac = (half == 1) ? 0.0 : double(i)/double(half-1); + const double period = min_p * std::pow(max_p/min_p, frac); + const double s = (2.0*M_PI/period)*t; out[i] = (float) std::sin(s); - out[half + i] = (float) std::cos(s); + out[half+i] = (float) std::cos(s); } return out; } @@ -86,8 +86,8 @@ inline void build_causal_mask(int64_t seq, std::vector & out) { out.assign((size_t) seq * seq, 0.0f); const float NEG = -std::numeric_limits::infinity(); for (int64_t q = 0; q < seq; ++q) - for (int64_t kv = q + 1; kv < seq; ++kv) - out[q * seq + kv] = NEG; + for (int64_t kv = q+1; kv < seq; ++kv) + out[q * seq+kv] = NEG; } } // namespace vla diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 8be91c3..9dc7595 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -131,7 +131,7 @@ ggml_tensor * build_qwen2_layer(ggml_context * C, const Evo1ModelArch & m, const ggml_tensor * qmask = nullptr) { const int64_t hd = m.lm_head_dim, n_q = m.n_q, n_kv = m.n_kv, hq = n_q * hd; const ggml_type at = m.act_type; - const float scale = 1.0f / std::sqrt((float) hd); + const float scale = 1.0f/std::sqrt((float) hd); ggml_tensor * h_n1 = ggml_mul(C, ggml_rms_norm(C, h, m.lm_rms_eps), w.attn_norm); ggml_tensor * qp = ggml_add(C, mm_act(C, w.Wq, h_n1, at), w.bq); ggml_tensor * kp = ggml_add(C, mm_act(C, w.Wk, h_n1, at), w.bk); @@ -167,17 +167,17 @@ bool preprocess_image_chw(const ImageView & v, int64_t side, std::vector std::fprintf(stderr, "vla(evo1): image view is %dx%d, expected %lldx%lld\n", v.w, v.h, (long long) side, (long long) side); return false; } - out.assign((size_t) 3 * side * side, 0.0f); + out.assign((size_t) 3*side * side, 0.0f); for (int64_t h = 0; h < side; ++h) for (int64_t w = 0; w < side; ++w) for (int64_t c = 0; c < 3; ++c) { float px; if (v.format == PixelFormat::U8) { - px = ((const uint8_t *) v.data)[(h * side + w) * 3 + c] / 255.0f; + px = ((const uint8_t *) v.data)[(h * side+w)*3+c]/255.0f; } else { - px = ((const float *) v.data)[(h * side + w) * 3 + c]; + px = ((const float *) v.data)[(h * side+w)*3+c]; } - out[c * side * side + h * side + w] = (px - MEAN[c]) / STD[c]; + out[c * side * side+h * side+w] = (px-MEAN[c])/STD[c]; } return true; } @@ -220,16 +220,16 @@ ggml_tensor * evo1_flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k ggml_tensor * build_internvit_layer(ggml_context * C, const Evo1ModelArch & m, const ViTLayerW & w, ggml_tensor * x, int64_t N) { - const int64_t H = m.vit_hidden, n_heads = m.vit_heads, hd = H / n_heads; + const int64_t H = m.vit_hidden, n_heads = m.vit_heads, hd = H/n_heads; const ggml_type at = m.act_type; - const float scale = 1.0f / std::sqrt((float) hd); + const float scale = 1.0f/std::sqrt((float) hd); ggml_tensor * x_n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, m.vit_ln_eps), w.n1w), w.n1b); ggml_tensor * qkv = ggml_add(C, mm_act(C, w.Wqkv, x_n1, at), w.bqkv); // one cast of the packed QKV rather than three of its slices qkv = as_type(C, qkv, GGML_TYPE_F32); - ggml_tensor * q = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 0 * H * ggml_element_size(qkv))); - ggml_tensor * k = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 1 * H * ggml_element_size(qkv))); - ggml_tensor * v = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 2 * H * ggml_element_size(qkv))); + ggml_tensor * q = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 0*H * ggml_element_size(qkv))); + ggml_tensor * k = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 1*H * ggml_element_size(qkv))); + ggml_tensor * v = ggml_cont(C, ggml_view_2d(C, qkv, H, N, qkv->nb[1], 2*H * ggml_element_size(qkv))); ggml_tensor * Q = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, q, hd, n_heads, N), 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, k, hd, n_heads, N), 0, 2, 1, 3)); ggml_tensor * att; @@ -253,8 +253,8 @@ ggml_tensor * build_internvit_layer(ggml_context * C, const Evo1ModelArch & m, c } ggml_tensor * build_internvit_view(ggml_context * C, const Evo1ModelArch & m, ggml_tensor * pixels) { - const int64_t H = m.vit_hidden, grid = m.image_size / m.patch_size, n_patches = grid * grid, n_tok = n_patches + 1; - const int64_t shuf_c = H * 4, sgrid = grid / 2; + const int64_t H = m.vit_hidden, grid = m.image_size/m.patch_size, n_patches = grid * grid, n_tok = n_patches+1; + const int64_t shuf_c = H*4, sgrid = grid/2; ggml_tensor * conv = ggml_conv_2d(C, m.vit_patch_w, pixels, (int) m.patch_size, (int) m.patch_size, 0, 0, 1, 1); ggml_tensor * patches = ggml_add(C, ggml_cont(C, ggml_transpose(C, ggml_reshape_2d(C, conv, n_patches, H))), m.vit_patch_b); @@ -267,7 +267,7 @@ ggml_tensor * build_internvit_view(ggml_context * C, const Evo1ModelArch & m, gg x = build_internvit_layer(C, m, m.vit[i], x, n_tok); ggml_tensor * pnc = ggml_cont(C, ggml_view_2d(C, x, H, n_patches, x->nb[1], x->nb[1])); - ggml_tensor * s1 = ggml_reshape_3d(C, pnc, 2 * H, sgrid, grid); + ggml_tensor * s1 = ggml_reshape_3d(C, pnc, 2*H, sgrid, grid); ggml_tensor * s2 = ggml_cont(C, ggml_permute(C, s1, 0, 2, 1, 3)); ggml_tensor * s3 = ggml_reshape_3d(C, s2, shuf_c, sgrid, sgrid); ggml_tensor * s4 = ggml_cont(C, ggml_permute(C, s3, 0, 2, 1, 3)); @@ -287,7 +287,7 @@ ggml_tensor * inproj_split_b(ggml_context * C, ggml_tensor * bin, int64_t E, int } bool load_config(const gguf_reader & g, Evo1ModelArch & m, Config & cfg) { - auto u = [&](const char * k, int64_t & dst) { if (g.has((std::string("evo1.") + k).c_str())) dst = g.u32((std::string("evo1.") + k).c_str()); }; + auto u = [&](const char * k, int64_t & dst) { if (g.has((std::string("evo1.")+k).c_str())) dst = g.u32((std::string("evo1.")+k).c_str()); }; u("lm_hidden", m.lm_hidden); u("lm_layers_used", m.lm_layers); u("lm_q_heads", m.n_q); u("lm_kv_heads", m.n_kv); u("lm_head_dim", m.lm_head_dim); u("lm_inter", m.lm_inter); u("embed_dim", m.embed_dim); u("dit_layers", m.dit_layers); u("dit_heads", m.dit_heads); u("mlp_head_hidden", m.mlp_head_hidden); u("horizon", m.horizon); u("per_action_dim", m.per_a); @@ -314,7 +314,7 @@ bool load_config(const gguf_reader & g, Evo1ModelArch & m, Config & cfg) { } // predict() reads action_dim noise floats; the server sizes noise as // horizon*per_action_dim, so they must agree or a client noise buffer underruns. - if (m.action_dim != m.horizon * m.per_a) { + if (m.action_dim != m.horizon*m.per_a) { std::fprintf(stderr, "vla(evo1): action_dim (%lld) != horizon (%lld) * per_action_dim (%lld)\n", (long long) m.action_dim, (long long) m.horizon, (long long) m.per_a); return false; } @@ -402,7 +402,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, } } - ggml_init_params wp = { (size_t) 32 * 1024 * 1024, + ggml_init_params wp = { (size_t) 32*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { @@ -493,7 +493,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, return nullptr; std::printf("vla(evo1): weights resident in %.2f GiB (%s)%s\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0 * 1024.0), + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0*1024.0), dtype_name(m->matmul_type), m->have_vision ? " - incl. InternViT vision tower" : " - vision tower NOT loaded (precomputed_img_emb required)"); @@ -538,7 +538,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { // // The branches are independent, so the arithmetic per view is unchanged // - only the submission pattern differs. - const size_t want_arena = (size_t) 32 * 1024 * 1024 * (size_t) std::max(n_views, 1); + const size_t want_arena = (size_t) 32*1024*1024*(size_t) std::max(n_views, 1); if (want_arena > vision_arena) { vision_scratch.release(); vision_arena = want_arena; @@ -552,7 +552,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { t_ie[v] = build_internvit_view(VC, *this, t_px[v]); ggml_set_output(t_ie[v]); } - ggml_cgraph * vg = ggml_new_graph_custom(VC, (size_t) 8192 * std::max(n_views, 1), false); + ggml_cgraph * vg = ggml_new_graph_custom(VC, (size_t) 8192*std::max(n_views, 1), false); for (int64_t v = 0; v < n_views; ++v) ggml_build_forward_expand(vg, t_ie[v]); if (!vision_scratch.alloc(backend, vg)) { @@ -571,10 +571,10 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { return {}; } for (int64_t v = 0; v < n_views; ++v) { - ggml_backend_tensor_get(t_ie[v], img_emb_host.data() + v * num_image_token * lm_hidden, + ggml_backend_tensor_get(t_ie[v], img_emb_host.data()+v * num_image_token * lm_hidden, 0, ggml_nbytes(t_ie[v])); } - stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); img_emb_ptr = img_emb_host.data(); } else { std::fprintf(stderr, "vla(evo1): no images and no precomputed_img_emb in the request\n"); @@ -621,8 +621,8 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { for (int64_t p = 0; p < SEQ; ++p) { if (input_ids[p] == (int32_t) img_ctx_id) { if (img_idx >= n_img_tokens) { std::fprintf(stderr, "vla(evo1): more IMG_CTX tokens than ViT embeds\n"); return {}; } - std::memcpy(inputs_embeds.data() + p * lm_hidden, - img_emb_ptr + img_idx * lm_hidden, lm_hidden * sizeof(float)); + std::memcpy(inputs_embeds.data()+p * lm_hidden, + img_emb_ptr+img_idx * lm_hidden, lm_hidden * sizeof(float)); ++img_idx; } } @@ -656,7 +656,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { continue; } const float sv = in.state ? in.state[i] : 0.0f; - float xn = 2.0f * (sv - lo) / (hi - lo + norm_eps_denom) - 1.0f; + float xn = 2.0f * (sv-lo)/(hi-lo+norm_eps_denom)-1.0f; if (xn < -1.0f) xn = -1.0f; if (xn > 1.0f) @@ -666,23 +666,23 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { std::vector x_init((size_t) action_dim); if (in.noise) { - std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); + std::memcpy(x_init.data(), in.noise, x_init.size()*sizeof(float)); } else { // Evo1 is trained with uniform[-1,1] noise uint64_t s = 0xE701ACE5ULL ^ (uint64_t) std::chrono::steady_clock::now().time_since_epoch().count(); for (auto & v : x_init) { - s = s * 6364136223846793005ULL + 1442695040888963407ULL; - v = ((float) (uint32_t) (s >> 32) / 2147483648.0f) - 1.0f; // uniform[-1,1) + s = s*6364136223846793005ULL+1442695040888963407ULL; + v = ((float) (uint32_t) (s >> 32)/2147483648.0f)-1.0f; // uniform[-1,1) } } // LM + DiT graph depends only on the padded length and step count. const MainKey mkey{ SEQ, num_steps }; - const bool built = main_graph.ensure(backend, mkey, (size_t) 96 * 1024 * 1024, + const bool built = main_graph.ensure(backend, mkey, (size_t) 96*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { - const int64_t E = embed_dim, hd_dit = E / dit_heads; - const float scale_dit = 1.0f / std::sqrt((float) hd_dit); - const int64_t Nctx = SEQ + 1; + const int64_t E = embed_dim, hd_dit = E/dit_heads; + const float scale_dit = 1.0f/std::sqrt((float) hd_dit); + const int64_t Nctx = SEQ+1; ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, lm_hidden, SEQ); ggml_set_input(t_embeds); ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, SEQ); ggml_set_input(t_pos); @@ -755,10 +755,10 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { return as_type(C, ggml_add(C, mm_act(C, head_W2, mh, at), head_b2), GGML_TYPE_F32); }; - const float dt = 1.0f / (float) num_steps; + const float dt = 1.0f/(float) num_steps; ggml_tensor * x_action = t_x; for (int64_t step = 0; step < num_steps; ++step) { - const int64_t time_index = (int64_t) ((double) step / (double) num_steps * 1000.0); + const int64_t time_index = (int64_t) ((double) step/(double) num_steps*1000.0); ggml_tensor * x_seq = ggml_reshape_2d(C, x_action, per_a, horizon); ggml_tensor * x_seq_masked = as_type(C, ggml_mul(C, x_seq, t_amask), at); ggml_tensor * v_t = denoise(x_seq_masked, time_index); @@ -790,7 +790,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } { std::vector mk((size_t) SEQ * SEQ); const float NEG = -std::numeric_limits::infinity(); - for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ + kv] = (kv <= q && attn_ok[kv]) ? 0.0f : NEG; + for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ+kv] = (kv <= q && attn_ok[kv]) ? 0.0f : NEG; ggml_backend_tensor_set(t_lmmask, mk.data(), 0, ggml_nbytes(t_lmmask)); } ggml_backend_tensor_set(t_state, state_norm.data(), 0, ggml_nbytes(t_state)); ggml_backend_tensor_set(t_x, x_init.data(), 0, ggml_nbytes(t_x)); @@ -806,18 +806,18 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(evo1): ggml_backend_graph_compute failed (%d)\n", (int) st); return {}; } - stats.ms_inference = std::chrono::duration(tc1 - tc0).count(); + stats.ms_inference = std::chrono::duration(tc1-tc0).count(); std::vector x_final((size_t) action_dim); - ggml_backend_tensor_get(x_action, x_final.data(), 0, x_final.size() * sizeof(float)); + ggml_backend_tensor_get(x_action, x_final.data(), 0, x_final.size()*sizeof(float)); std::vector out((size_t) horizon * per_a); for (int64_t hstep = 0; hstep < horizon; ++hstep) for (int64_t c = 0; c < per_a; ++c) { - const double a = (double) x_final[hstep * per_a + c]; - out[hstep * per_a + c] = (float) ((a + 1.0) / 2.0 * ((double) action_max[c] - (double) action_min[c] + (double) norm_eps_denom) + (double) action_min[c]); + const double a = (double) x_final[hstep * per_a+c]; + out[hstep * per_a+c] = (float) ((a+1.0)/2.0*((double) action_max[c]-(double) action_min[c]+(double) norm_eps_denom)+(double) action_min[c]); } - stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now()-t0).count(); return out; } diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index 4ae4c2b..afdd8cb 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -146,8 +146,8 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { // merge_block_coords only enumerates the patch grid exactly when the spatial // merge divides it; otherwise it emits rows past the position table. - if (m.patch_size <= 0 || m.spatial_merge <= 0 || m.image_target_size % m.patch_size != 0 || - (m.image_target_size / m.patch_size) % m.spatial_merge != 0) { + if (m.patch_size <= 0 || m.spatial_merge <= 0 || m.image_target_size%m.patch_size != 0 || + (m.image_target_size/m.patch_size)%m.spatial_merge != 0) { std::fprintf(stderr, "vla(gr00tn1d7): image %lld / patch %lld / merge %lld do not divide evenly\n", (long long) m.image_target_size, (long long) m.patch_size, (long long) m.spatial_merge); return false; @@ -198,10 +198,10 @@ bool load_config(const gguf_reader & g, Gr00tN1d7ModelArch & m, Config & cfg) { { const std::string js = g.str(fk("embodiment_id_mapping")); auto lookup = [&](const char * key) -> long { - const std::string k = std::string("\"") + key + "\""; + const std::string k = std::string("\"")+key + "\""; size_t p = js.find(k); if (p == std::string::npos) return -1; - p = js.find(':', p + k.size()); if (p == std::string::npos) return -1; - return std::strtol(js.c_str() + p + 1, nullptr, 10); + p = js.find(':', p+k.size()); if (p == std::string::npos) return -1; + return std::strtol(js.c_str()+p+1, nullptr, 10); }; long ls = lookup("libero_sim"); if (ls >= 0) m.aex.embodiment_id = ls; if (const char * e = std::getenv("VLA_GR00T_EMBODIMENT")) { @@ -318,8 +318,8 @@ bool Gr00tN1d7ModelArch::build_caches() { if (caches_ready) return true; const int64_t side = image_target_size, ps = patch_size, m2 = spatial_merge; - const int64_t grid = side / ps; - const int64_t hd_vit = vit_hidden / vit_heads; + const int64_t grid = side/ps; + const int64_t hd_vit = vit_hidden/vit_heads; const int64_t num_side = (int64_t) std::lround(std::sqrt((double) vit_num_pos)); const int64_t E = in_embed_dim, AH = action_horizon; @@ -338,7 +338,7 @@ bool Gr00tN1d7ModelArch::build_caches() { c_tau.assign((size_t) num_steps, {}); c_tproj.assign((size_t) num_steps, {}); for (int64_t s = 0; s < num_steps; ++s) { - const int64_t bucket = (int64_t) ((double) s / (double) num_steps * (double) num_buckets); + const int64_t bucket = (int64_t) ((double) s/(double) num_steps * (double) num_buckets); action_sinusoid(bucket, E, AH, c_tau[(size_t) s]); timesteps_proj(bucket, c_tproj[(size_t) s]); } @@ -353,11 +353,11 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { const int64_t H = lm_hidden, E = in_embed_dim; const int64_t side = image_target_size; const int64_t ps = patch_size, m2 = spatial_merge; - const int64_t grid = side / ps; + const int64_t grid = side/ps; const int64_t n_patches = grid * grid; - const int64_t K = (grid / m2) * (grid / m2); - const int64_t hd_vit = vit_hidden / vit_heads; - const int64_t AD = action_dim, AH = action_horizon, Nsa = 1 + AH; + const int64_t K = (grid/m2)*(grid/m2); + const int64_t hd_vit = vit_hidden/vit_heads; + const int64_t AD = action_dim, AH = action_horizon, Nsa = 1+AH; const bool do_dump = (std::getenv("VLA_GR00T_N17_DUMP") != nullptr); if (!caches_ready) { std::fprintf(stderr, "vla(gr00tn1d7): caches not ready\n"); return {}; } @@ -378,7 +378,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { for (int j = 0; j < 3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); - ggml_context * VC = vision_scratch.reset((size_t) 512 * 1024 * 1024); + ggml_context * VC = vision_scratch.reset((size_t) 512*1024*1024); if (!VC) { std::fprintf(stderr, "vla(gr00tn1d7): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_patches = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_patch_flat, n_patches); ggml_set_input(t_patches); ggml_tensor * t_pos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_hidden, n_patches); ggml_set_input(t_pos); @@ -425,11 +425,11 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { vok = false; break; } - ggml_backend_tensor_get(vit_embeds, img_emb_host.data() + v * K * H, 0, ggml_nbytes(vit_embeds)); + ggml_backend_tensor_get(vit_embeds, img_emb_host.data()+v * K * H, 0, ggml_nbytes(vit_embeds)); for (int j = 0; j < 3; ++j) - ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); + ggml_backend_tensor_get(ds_out[j], ds_host[j].data()+v * K * H, 0, ggml_nbytes(ds_out[j])); } - stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); if (!vok) return {}; img_emb_ptr = img_emb_host.data(); } else { @@ -443,9 +443,9 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_img_slots; if (n_img_slots == n_img) { - input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); + input_ids.assign(in.lang_tokens, in.lang_tokens+in.n_lang); } else if (n_img_slots == 0) { - input_ids.reserve(n_img + in.n_lang); + input_ids.reserve(n_img+in.n_lang); for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); for (int j = 0; j < in.n_lang; ++j) @@ -463,12 +463,12 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { { int64_t k = 0; for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { if (k >= n_img) { std::fprintf(stderr, "vla(gr00tn1d7): more tokens than ViT embeds\n"); return {}; } - std::memcpy(inputs_embeds.data() + p * H, img_emb_ptr + k * H, H * sizeof(float)); ++k; + std::memcpy(inputs_embeds.data()+p * H, img_emb_ptr+k * H, H * sizeof(float)); ++k; } } std::vector image_pos_idx, text_pos_idx; - image_pos_idx.reserve((size_t) n_img); text_pos_idx.reserve((size_t) (SEQ - n_img)); + image_pos_idx.reserve((size_t) n_img); text_pos_idx.reserve((size_t) (SEQ-n_img)); for (int64_t p = 0; p < SEQ; ++p) { if (input_ids[p] == (int32_t) image_token_index) image_pos_idx.push_back((int32_t) p); @@ -485,14 +485,14 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { if (inject_deepstack) for (int j = 0; j < 3; ++j) { ds_pad[j].assign((size_t) SEQ * H, 0.0f); for (int64_t k = 0; k < n_img; ++k) { - std::memcpy(ds_pad[j].data() + (size_t) image_pos_idx[k] * H, - ds_host[j].data() + (size_t) k * H, H * sizeof(float)); + std::memcpy(ds_pad[j].data()+(size_t) image_pos_idx[k]*H, + ds_host[j].data()+(size_t) k * H, H * sizeof(float)); } } std::vector x_init((size_t) AH * AD); if (in.noise) - std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); + std::memcpy(x_init.data(), in.noise, x_init.size()*sizeof(float)); else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); @@ -510,10 +510,10 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * eagle = nullptr, * vl_embs = nullptr; std::vector lm_h_dump, vlsa_dump; const MainKey mkey{ SEQ, n_img, SEQ_TXT, num_steps, inject_deepstack }; - const bool built = mg.ensure(backend, mkey, (size_t) 256 * 1024 * 1024, + const bool built = mg.ensure(backend, mkey, (size_t) 256*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); - ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, 4 * SEQ); ggml_set_input(t_pos); + ggml_tensor * t_pos = ggml_new_tensor_1d(C, GGML_TYPE_I32, 4*SEQ); ggml_set_input(t_pos); ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1);ggml_set_input(t_state); ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); @@ -567,14 +567,14 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * state_features = cat_linear(C, aex.se_l2W, aex.se_l2b, aex.embodiment_id, ggml_relu(C, cat_linear(C, aex.se_l1W, aex.se_l1b, aex.embodiment_id, t_state))); - const float dt = 1.0f / (float) num_steps; - const int64_t every2 = 2 * attend_text_every_n; + const float dt = 1.0f/(float) num_steps; + const int64_t every2 = 2*attend_text_every_n; std::vector Kc(dit_layers, nullptr), Vc(dit_layers, nullptr); for (int64_t i = 0; i < dit_layers; ++i) { - if (dit_interleave && (i % 2 == 1)) + if (dit_interleave && (i%2 == 1)) continue; - ggml_tensor * enc = (i % every2 == 0) ? vl_txt : vl_img; + ggml_tensor * enc = (i%every2 == 0) ? vl_txt : vl_img; dit.kv(C, dit.blk[i], enc, &Kc[i], &Vc[i]); } @@ -588,9 +588,9 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * hh = sa; for (int64_t i = 0; i < dit_layers; ++i) { ggml_tensor * enc; - if (dit_interleave && (i % 2 == 1)) + if (dit_interleave && (i%2 == 1)) enc = nullptr; - else if (i % every2 == 0) enc = vl_txt; + else if (i%every2 == 0) enc = vl_txt; else enc = vl_img; hh = dit.block(C, dit.blk[i], hh, temb, enc, Kc[i], Vc[i]); @@ -601,7 +601,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * h_mod = ggml_add(C, ggml_add(C, hn, ggml_mul(C, hn, sc)), sh); ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, dit.po2W, h_mod), dit.po2b); ggml_tensor * pred = cat_linear(C, aex.ad_l2W, aex.ad_l2b, aex.embodiment_id, ggml_relu(C, cat_linear(C, aex.ad_l1W, aex.ad_l1b, aex.embodiment_id, model_output))); - ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t) (Nsa - AH) * pred->nb[1])); + ggml_tensor * vel = ggml_cont(C, ggml_view_2d(C, pred, AD, AH, pred->nb[1], (size_t) (Nsa-AH)*pred->nb[1])); actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); } ggml_set_name(actions, "action_pred"); ggml_set_output(actions); @@ -626,10 +626,10 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); { - const int64_t llm_grid_h = image_target_size / patch_size / spatial_merge; + const int64_t llm_grid_h = image_target_size/patch_size/spatial_merge; const int64_t llm_grid_w = llm_grid_h; - std::vector pp((size_t) 4 * SEQ, 0); + std::vector pp((size_t) 4*SEQ, 0); int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; @@ -638,12 +638,12 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { break; } const int64_t text_end = (img_start < 0) ? SEQ : img_start; - const int64_t text_len = text_end - st; + const int64_t text_len = text_end-st; for (int64_t i = 0; i < text_len; ++i) { - const int32_t p = (int32_t) (i + st_idx); - pp[0 * SEQ + (st + i)] = p; - pp[1 * SEQ + (st + i)] = p; - pp[2 * SEQ + (st + i)] = p; + const int32_t p = (int32_t) (i+st_idx); + pp[0*SEQ+(st+i)] = p; + pp[1*SEQ+(st+i)] = p; + pp[2*SEQ+(st+i)] = p; } if (img_start < 0) { st_idx += text_len; @@ -653,35 +653,35 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { int64_t img_end = img_start; while (img_end < SEQ && input_ids[img_end] == (int32_t) image_token_index) ++img_end; - const int64_t n_img_tokens = img_end - img_start; - if (n_img_tokens % (llm_grid_h * llm_grid_w) != 0) { + const int64_t n_img_tokens = img_end-img_start; + if (n_img_tokens%(llm_grid_h * llm_grid_w) != 0) { std::fprintf(stderr, "vla(gr00tn1d7): image run length %lld not a multiple of %lld (post-merge grid)\n", (long long) n_img_tokens, (long long) (llm_grid_h * llm_grid_w)); mg.release(); return {}; } - const int64_t this_t = n_img_tokens / (llm_grid_h * llm_grid_w); - const int64_t image_offset = text_len + st_idx; + const int64_t this_t = n_img_tokens/(llm_grid_h * llm_grid_w); + const int64_t image_offset = text_len+st_idx; for (int64_t tt = 0; tt < this_t; ++tt) { for (int64_t hh = 0; hh < llm_grid_h; ++hh) { for (int64_t ww = 0; ww < llm_grid_w; ++ww) { - const int64_t k = (tt * llm_grid_h + hh) * llm_grid_w + ww; - const int64_t tok = img_start + k; - pp[0 * SEQ + tok] = (int32_t) (image_offset + tt); - pp[1 * SEQ + tok] = (int32_t) (image_offset + hh); - pp[2 * SEQ + tok] = (int32_t) (image_offset + ww); + const int64_t k = (tt * llm_grid_h+hh)*llm_grid_w+ww; + const int64_t tok = img_start+k; + pp[0*SEQ+tok] = (int32_t) (image_offset+tt); + pp[1*SEQ+tok] = (int32_t) (image_offset+hh); + pp[2*SEQ+tok] = (int32_t) (image_offset+ww); } } } - int64_t max_image_pos = this_t - 1; - if (llm_grid_h - 1 > max_image_pos) - max_image_pos = llm_grid_h - 1; - if (llm_grid_w - 1 > max_image_pos) - max_image_pos = llm_grid_w - 1; - st_idx = image_offset + max_image_pos + 1; + int64_t max_image_pos = this_t-1; + if (llm_grid_h-1 > max_image_pos) + max_image_pos = llm_grid_h-1; + if (llm_grid_w-1 > max_image_pos) + max_image_pos = llm_grid_w-1; + st_idx = image_offset+max_image_pos+1; st = img_end; } - std::memcpy(pp.data() + (size_t) 3 * SEQ, pp.data() + (size_t) 0 * SEQ, (size_t) SEQ * sizeof(int32_t)); + std::memcpy(pp.data()+(size_t) 3*SEQ, pp.data()+(size_t) 0*SEQ, (size_t) SEQ * sizeof(int32_t)); ggml_backend_tensor_set(t_pos, pp.data(), 0, ggml_nbytes(t_pos)); } if (c_mask_seq != SEQ) { @@ -709,16 +709,16 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { const ggml_status st = ggml_backend_graph_compute(backend, gf); const auto tc1 = std::chrono::steady_clock::now(); if (st != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d7): graph compute failed (%d)\n", (int) st); mg.release(); return {}; } - stats.ms_inference = std::chrono::duration(tc1 - tc0).count(); + stats.ms_inference = std::chrono::duration(tc1-tc0).count(); std::vector out((size_t) AH * AD); - ggml_backend_tensor_get(actions, out.data(), 0, out.size() * sizeof(float)); + ggml_backend_tensor_get(actions, out.data(), 0, out.size()*sizeof(float)); if (const char * dump = std::getenv("VLA_GR00T_N17_DUMP")) { auto dump_t = [&](const char * name, ggml_tensor * t) { const int64_t n0 = t->ne[0], n1 = t->ne[1]; - std::vector buf((size_t) n0 * n1); - ggml_backend_tensor_get(t, buf.data(), 0, buf.size() * sizeof(float)); + std::vector buf((size_t) n0*n1); + ggml_backend_tensor_get(t, buf.data(), 0, buf.size()*sizeof(float)); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump, name, (long long) n0, (long long) n1); FILE * fp = std::fopen(path, "wb"); if (fp) { @@ -747,7 +747,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump, name, (long long) n0, (long long) n1); FILE * fp = std::fopen(path, "wb"); if (fp) { - std::fwrite(data, sizeof(float), (size_t) n0 * n1, fp); + std::fwrite(data, sizeof(float), (size_t) n0*n1, fp); std::fclose(fp); std::fprintf(stderr, "vla(gr00tn1d7): dumped %s shape=(%lld,%lld) to %s\n", name, (long long) n1, (long long) n0, path); } @@ -755,19 +755,19 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { for (int64_t v = 0; v < n_views; ++v) { char nm[32]; std::snprintf(nm, sizeof(nm), "ds0_view%lld", (long long) v); - dump_host(nm, ds_host[0].data() + v * K * H, H, K); + dump_host(nm, ds_host[0].data()+v * K * H, H, K); std::snprintf(nm, sizeof(nm), "ds1_view%lld", (long long) v); - dump_host(nm, ds_host[1].data() + v * K * H, H, K); + dump_host(nm, ds_host[1].data()+v * K * H, H, K); std::snprintf(nm, sizeof(nm), "ds2_view%lld", (long long) v); - dump_host(nm, ds_host[2].data() + v * K * H, H, K); + dump_host(nm, ds_host[2].data()+v * K * H, H, K); std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); - dump_host(nm, img_emb_host.data() + v * K * H, H, K); + dump_host(nm, img_emb_host.data()+v * K * H, H, K); } } } if (!use_cache) mg.release(); - stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now()-t0).count(); return out; } diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index 11343af..2e0ae00 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -55,8 +55,8 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, } else { size_t b = js.find('{'); size_t q = js.find('"', b); - size_t qe = js.find('"', q + 1); - suite = js.substr(q + 1, qe - q - 1); suite_pos = q; + size_t qe = js.find('"', q+1); + suite = js.substr(q+1, qe-q-1); suite_pos = q; } if (suite_pos == std::string::npos) { std::fprintf(stderr, "vla(openvla_oft): suite '%s' not in stats\n", suite.c_str()); @@ -70,7 +70,7 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, size_t lb = js.find('[', k); size_t rb = js.find(']', lb); if (lb == std::string::npos || rb == std::string::npos) return false; - out.clear(); size_t p = lb + 1; + out.clear(); size_t p = lb+1; while (p < rb) { while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) ++p; @@ -82,7 +82,7 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, p += t ? 4 : 5; } else { - out.push_back(std::strtof(js.c_str() + p, nullptr)); + out.push_back(std::strtof(js.c_str()+p, nullptr)); while (p < rb && js[p] != ',') ++p; } @@ -189,7 +189,7 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path // (modeling_prismatic.py:891), which is what act0 below does. U("openvla_oft.tokens.stop_id",m->stop_id); if (m->lm_head_dim==0) - m->lm_head_dim = m->lm_hidden / m->n_q; + m->lm_head_dim = m->lm_hidden/m->n_q; if (g.has("openvla_oft.statistics_json")) { if (!parse_stats(g.str("openvla_oft.statistics_json"), m->action_dim, m->q01, m->q99, m->unnorm_mask, m->suite)) @@ -318,10 +318,10 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { return {}; } const int64_t n_act = chunk * action_dim; - const int64_t NUM_PATCHES = NPATCH + 1; - const int64_t NUM_PROMPT_TOKENS = L - 1; - const int64_t ACT_START = NUM_PATCHES + NUM_PROMPT_TOKENS; - const int64_t SEQ = 1 + NUM_PATCHES + (L-1) + n_act + 1; + const int64_t NUM_PATCHES = NPATCH+1; + const int64_t NUM_PROMPT_TOKENS = L-1; + const int64_t ACT_START = NUM_PATCHES+NUM_PROMPT_TOKENS; + const int64_t SEQ = 1+NUM_PATCHES+(L-1)+n_act+1; const auto ti=clock::now(); // LM + action head graph depends only on the sequence layout. const MainKey mkey{ SEQ, n_views, L }; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index bba826c..5e95ee7 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -59,7 +59,7 @@ bool is_gemma_norm(const std::string & name) { bool ends_with(const std::string & s, const char * sfx) { const size_t n = std::strlen(sfx); - return s.size() >= n && s.compare(s.size() - n, n, sfx) == 0; + return s.size() >= n && s.compare(s.size()-n, n, sfx) == 0; } } @@ -134,7 +134,7 @@ namespace { ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps, ggml_type at) { - const float scale = 1.0f / std::sqrt((float) head_dim); + const float scale = 1.0f/std::sqrt((float) head_dim); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); ggml_tensor * q = as_type(C, ggml_add(C, mm_act(C, w.Wq, n1, at), w.bq), GGML_TYPE_F32); ggml_tensor * k = as_type(C, ggml_add(C, mm_act(C, w.Wk, n1, at), w.bk), GGML_TYPE_F32); @@ -207,7 +207,7 @@ ggml_tensor * build_gemma_layer( V_full = ggml_concat(ctx, cached_V, v_h, 2); } - const float scale = 1.f / std::sqrt((float) hd); + const float scale = 1.f/std::sqrt((float) hd); ggml_tensor * Q = ggml_cont(ctx, ggml_permute(ctx, q_rope, 0, 2, 1, 3)); ggml_tensor * K = ggml_cont(ctx, ggml_permute(ctx, K_full, 0, 2, 1, 3)); ggml_tensor * att_pre; @@ -292,7 +292,7 @@ bool load_config(const gguf_reader & g, Config & cfg) { cfg.n_state = 1; cfg.n_img = 256; cfg.q_full_dim = cfg.n_q_heads * cfg.head_dim; - cfg.kv_full_dim = cfg.n_kv_heads * cfg.head_dim; + cfg.kv_full_dim = cfg.n_kv_heads*cfg.head_dim; cfg.self_attn_every_n = 0; cfg.rms_eps = g.has("pi0.rms_norm_eps") ? g.f32("pi0.rms_norm_eps") : 1e-6f; cfg.norm_eps = g.has("pi0.norm_eps") ? g.f32("pi0.norm_eps") : 1e-8f; @@ -321,7 +321,7 @@ bool load_stats(gguf_reader & g, Pi0ModelArch & m) { return; } const std::vector identity = dst; - if (!g.read_raw(name, dst.data(), dst.size() * sizeof(float))) { + if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { // A short read leaves dst half-overwritten. dst = identity; std::printf("vla(pi0): %s read failed - identity\n", name); @@ -412,7 +412,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, vu("pi0.patch_size", m->vit_patch_size); vu("pi0.n_img_tokens", m->vit_n_tokens); if (g.has("pi0.vit_ln_eps")) m->vit_ln_eps = g.f32("pi0.vit_ln_eps"); - const int64_t grid = m->vit_image_size / m->vit_patch_size; + const int64_t grid = m->vit_image_size/m->vit_patch_size; if (grid * grid != m->vit_n_tokens || m->vit_n_tokens != cfg.n_img) { std::fprintf(stderr, "vla(pi0): vit geometry mismatch (grid^2=%lld n_img_tokens=%lld cfg.n_img=%lld)\n", (long long) (grid * grid), (long long) m->vit_n_tokens, (long long) cfg.n_img); @@ -421,7 +421,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, } { - ggml_init_params wp = { (size_t) 16 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 16*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(pi0): ggml_init(ctx_weights) failed\n"); @@ -464,30 +464,30 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { const int64_t hidden_pl = cfg.hidden; const int64_t hidden_ex = cfg.expert_h; const int64_t chunk = cfg.n_suffix; - const int64_t n_suf = 1 + chunk; + const int64_t n_suf = 1+chunk; const int64_t n_layers = cfg.n_layers; const int64_t max_sd = cfg.max_state_dim; const int64_t max_ad = cfg.max_action_dim; const int num_steps = cfg.num_steps; - const float dt = -1.0f / (float) num_steps; + const float dt = -1.0f/(float) num_steps; const float rope_base = cfg.rope_freq_base; std::vector img_emb_host; int64_t n_img_tokens = 0; if (in.precomputed_img_emb) { - n_img_tokens = (int64_t) in.n_img_views * cfg.n_img; + n_img_tokens = (int64_t) in.n_img_views*cfg.n_img; img_emb_host.assign(in.precomputed_img_emb, - in.precomputed_img_emb + (size_t) n_img_tokens * hidden_pl); + in.precomputed_img_emb+(size_t) n_img_tokens * hidden_pl); } else { if (in.n_images < 1 || !in.images) { std::fprintf(stderr, "vla(pi0): predict: no images and no precomputed_img_emb\n"); return {}; } - const int64_t K = vit_n_tokens, H = hidden_pl, grid = vit_image_size / vit_patch_size; - n_img_tokens = (int64_t) in.n_images * K; - img_emb_host.assign((size_t) in.n_images * K * H, 0.0f); + const int64_t K = vit_n_tokens, H = hidden_pl, grid = vit_image_size/vit_patch_size; + n_img_tokens = (int64_t) in.n_images*K; + img_emb_host.assign((size_t) in.n_images*K * H, 0.0f); - ggml_context * VC = vision_scratch.reset((size_t) 128 * 1024 * 1024); + ggml_context * VC = vision_scratch.reset((size_t) 128*1024*1024); if (!VC) { std::fprintf(stderr, "vla(pi0): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, vit_image_size, vit_image_size, 3); ggml_set_input(t_px); ggml_tensor * conv = ggml_conv_2d(VC, vit.patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); @@ -495,14 +495,14 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { // patch embed (conv_2d) stays F32; the tower runs in the activation dtype ggml_tensor * h = as_type(VC, ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos), act_type); for (int64_t i = 0; i < vit_layers; ++i) - h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps, act_type); + h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden/vit_heads, vit_hidden, vit_ln_eps, act_type); h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = mm_act(VC, mm_proj_w, h, act_type); if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); // read back to the host as F32 - ggml_tensor * vit_emb = as_type(VC, ggml_scale(VC, proj, 1.0f / std::sqrt((float) proj->ne[0])), GGML_TYPE_F32); + ggml_tensor * vit_emb = as_type(VC, ggml_scale(VC, proj, 1.0f/std::sqrt((float) proj->ne[0])), GGML_TYPE_F32); ggml_set_output(vit_emb); ggml_cgraph * vg = ggml_new_graph_custom(VC, 8192, false); @@ -521,9 +521,9 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(pi0): vision compute failed (view %d)\n", v); return {}; } - ggml_backend_tensor_get(vit_emb, img_emb_host.data() + (size_t) v * K * H, 0, ggml_nbytes(vit_emb)); + ggml_backend_tensor_get(vit_emb, img_emb_host.data()+(size_t) v * K * H, 0, ggml_nbytes(vit_emb)); } - stats.ms_vision = std::chrono::duration(clk::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(clk::now()-tv0).count(); } if (in.n_lang < 1 || !in.lang_tokens) { @@ -531,10 +531,10 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { return {}; } const int64_t n_lang = in.n_lang; - const int64_t n_prefix = n_img_tokens + n_lang; - const int64_t n_total = n_prefix + n_suf; + const int64_t n_prefix = n_img_tokens+n_lang; + const int64_t n_total = n_prefix+n_suf; - std::vector lang_ids(in.lang_tokens, in.lang_tokens + n_lang); + std::vector lang_ids(in.lang_tokens, in.lang_tokens+n_lang); std::vector lang_rows((size_t) n_lang * hidden_pl); { if (!io.fetch_rows_f32("token_embd.weight", lang_ids, lang_rows.data(), hidden_pl)) return {}; @@ -542,7 +542,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { // Prefix + expert graph depends only on the token counts and step count. const MainKey mkey{ n_img_tokens, n_lang, num_steps }; - const bool built = main_graph.ensure(backend, mkey, (size_t) 64 * 1024 * 1024, + const bool built = main_graph.ensure(backend, mkey, (size_t) 64*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { ggml_tensor * t_image_emb = ggml_new_tensor_2d(C, GGML_TYPE_F32, hidden_pl, n_img_tokens); ggml_set_input(t_image_emb); ggml_tensor * t_lang_emb = ggml_new_tensor_2d(C, GGML_TYPE_F32, hidden_pl, n_lang); ggml_set_input(t_lang_emb); @@ -619,7 +619,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { { std::vector pp(n_prefix); for (int64_t i = 0; i < n_prefix; ++i) pp[i] = (int32_t) i; ggml_backend_tensor_set(t_prefix_pos, pp.data(), 0, ggml_nbytes(t_prefix_pos)); - std::vector sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix + i); + std::vector sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix+i); ggml_backend_tensor_set(t_suffix_pos, sp.data(), 0, ggml_nbytes(t_suffix_pos)); } { @@ -628,13 +628,13 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { for (int64_t i = 0; i < max_sd; ++i) sh[i] = in.state ? in.state[i] : 0.f; for (int64_t i = 0; i < cfg.real_state_dim && i < max_sd; ++i) - sh[i] = (sh[i] - state_mean[i]) / (state_std[i] + cfg.norm_eps); + sh[i] = (sh[i]-state_mean[i])/(state_std[i]+cfg.norm_eps); ggml_backend_tensor_set(t_state, sh.data(), 0, ggml_nbytes(t_state)); } { std::vector x0h((size_t) max_ad * chunk); if (in.noise) - std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); + std::memcpy(x0h.data(), in.noise, x0h.size()*sizeof(float)); else { std::normal_distribution nd(0.f, 1.f); for (auto & v : x0h) @@ -651,39 +651,39 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { if (j < n_prefix) allowed = true; else { - const int64_t s = j - n_prefix; + const int64_t s = j-n_prefix; allowed = (i == 0) ? (s == 0) : true; } - mk[i * n_total + j] = allowed ? 0.f : -INFINITY; + mk[i * n_total+j] = allowed ? 0.f : -INFINITY; } ggml_backend_tensor_set(t_full_mask, mk.data(), 0, ggml_nbytes(t_full_mask)); } for (int s = 0; s < num_steps; ++s) { - const float timestep = 1.0f + (float) s * dt; + const float timestep = 1.0f+(float) s * dt; const std::vector tv = sinusoidal_time_emb(timestep, hidden_ex, cfg.min_period, cfg.max_period); std::vector tile((size_t) hidden_ex * chunk); for (int64_t c = 0; c < chunk; ++c) - std::memcpy(tile.data() + c * hidden_ex, tv.data(), hidden_ex * sizeof(float)); + std::memcpy(tile.data()+c * hidden_ex, tv.data(), hidden_ex * sizeof(float)); ggml_backend_tensor_set(t_time[s], tile.data(), 0, ggml_nbytes(t_time[s])); } const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); - stats.ms_inference = std::chrono::duration(clk::now() - ti0).count(); + stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); if (st != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(pi0): ggml_backend_graph_compute failed (%d)\n", (int) st); return {}; } std::vector out((size_t) chunk * max_ad); - ggml_backend_tensor_get(x_final, out.data(), 0, out.size() * sizeof(float)); + ggml_backend_tensor_get(x_final, out.data(), 0, out.size()*sizeof(float)); for (int64_t t = 0; t < chunk; ++t) { - float * row = out.data() + (size_t) t * max_ad; + float * row = out.data()+(size_t) t * max_ad; for (int64_t j = 0; j < max_ad; ++j) - row[j] = j < cfg.real_action_dim ? row[j] * (action_std[j] + cfg.norm_eps) + action_mean[j] : 0.0f; + row[j] = j < cfg.real_action_dim ? row[j]*(action_std[j]+cfg.norm_eps)+action_mean[j] : 0.0f; } - stats.ms_total = std::chrono::duration(clk::now() - t0).count(); + stats.ms_total = std::chrono::duration(clk::now()-t0).count(); return out; } diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 46fd095..1dab93f 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -64,7 +64,7 @@ struct ExpertLayerW { bool ends_with(const std::string & s, const char * sfx) { const size_t n = std::strlen(sfx); - return s.size() >= n && s.compare(s.size() - n, n, sfx) == 0; + return s.size() >= n && s.compare(s.size()-n, n, sfx) == 0; } bool starts_with(const std::string & s, const char * pfx) { const size_t n = std::strlen(pfx); @@ -139,7 +139,7 @@ namespace { // attention (nullptr mask), F32 score accumulation, tanh GELU FFN. ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) head_dim); + const float scale = 1.0f/std::sqrt((float) head_dim); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); @@ -197,7 +197,7 @@ ggml_tensor * build_vlm_layer( ggml_tensor * kq = ggml_mul_mat(ctx, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - const float scale = 1.f / std::sqrt((float) hd); + const float scale = 1.f/std::sqrt((float) hd); ggml_tensor * attn = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.f); ggml_tensor * kqv = ggml_mul_mat(ctx, V, attn); @@ -221,7 +221,7 @@ ggml_tensor * build_adarms( ggml_tensor * mod = ggml_add(ctx, ggml_mul_mat(ctx, dense_w, cond), dense_b); ggml_tensor * scale = ggml_view_1d(ctx, mod, h, 0); ggml_tensor * shift = ggml_view_1d(ctx, mod, h, (size_t) h * sizeof(float)); - ggml_tensor * gate = ggml_view_1d(ctx, mod, h, (size_t) 2 * h * sizeof(float)); + ggml_tensor * gate = ggml_view_1d(ctx, mod, h, (size_t) 2*h * sizeof(float)); ggml_tensor * normed = ggml_rms_norm(ctx, x, eps); ggml_tensor * out = ggml_add(ctx, @@ -270,7 +270,7 @@ ggml_tensor * build_expert_layer( ggml_tensor * kq = ggml_mul_mat(ctx, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - const float scale = 1.f / std::sqrt((float) hd); + const float scale = 1.f/std::sqrt((float) hd); ggml_tensor * attn = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.f); ggml_tensor * kqv = ggml_mul_mat(ctx, V, attn); @@ -328,7 +328,7 @@ bool load_config(const gguf_reader & g, Config & cfg) { cfg.n_state = 0; cfg.n_img = 256; cfg.q_full_dim = cfg.n_q_heads * cfg.head_dim; - cfg.kv_full_dim = cfg.n_kv_heads * cfg.head_dim; + cfg.kv_full_dim = cfg.n_kv_heads*cfg.head_dim; cfg.self_attn_every_n = 0; cfg.rms_eps = g.has("pi05.rms_norm_eps") ? g.f32("pi05.rms_norm_eps") : 1e-6f; cfg.norm_eps = g.has("pi05.norm_eps") ? g.f32("pi05.norm_eps") : 1e-8f; @@ -357,7 +357,7 @@ bool load_stats(gguf_reader & g, Pi05ModelArch & m) { return; } const std::vector identity = dst; - if (!g.read_raw(name, dst.data(), dst.size() * sizeof(float))) { + if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { // A short read leaves dst half-overwritten. dst = identity; std::printf("vla(pi05): %s read failed - identity\n", name); @@ -446,7 +446,7 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, vu("pi05.patch_size", m->vit_patch_size); vu("pi05.n_img_tokens", m->vit_n_tokens); if (g.has("pi05.vit_ln_eps")) m->vit_ln_eps = g.f32("pi05.vit_ln_eps"); - const int64_t grid = m->vit_image_size / m->vit_patch_size; + const int64_t grid = m->vit_image_size/m->vit_patch_size; if (grid * grid != m->vit_n_tokens || m->vit_n_tokens != cfg.n_img) { std::fprintf(stderr, "vla(pi05): vit geometry mismatch (grid^2=%lld n_img_tokens=%lld cfg.n_img=%lld)\n", (long long) (grid * grid), (long long) m->vit_n_tokens, (long long) cfg.n_img); @@ -455,7 +455,7 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, } { - ggml_init_params wp = { (size_t) 16 * 1024 * 1024, nullptr, true }; + ggml_init_params wp = { (size_t) 16*1024*1024, nullptr, true }; m->ctx_weights = ggml_init(wp); if (!m->ctx_weights) { std::fprintf(stderr, "vla(pi05): ggml_init(ctx_weights) failed\n"); @@ -554,38 +554,38 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { const int64_t n_layers = cfg.n_layers; const int64_t max_ad = cfg.max_action_dim; const int num_steps = cfg.num_steps; - const float dt = -1.0f / (float) num_steps; + const float dt = -1.0f/(float) num_steps; const float rope_base = cfg.rope_freq_base; std::vector img_emb_host; int64_t n_img_tokens = 0; if (in.precomputed_img_emb) { - n_img_tokens = (int64_t) in.n_img_views * cfg.n_img; + n_img_tokens = (int64_t) in.n_img_views*cfg.n_img; img_emb_host.assign(in.precomputed_img_emb, - in.precomputed_img_emb + (size_t) n_img_tokens * hidden_pl); + in.precomputed_img_emb+(size_t) n_img_tokens * hidden_pl); } else { if (in.n_images < 1 || !in.images) { std::fprintf(stderr, "vla(pi05): predict: no images and no precomputed_img_emb\n"); return {}; } - const int64_t K = vit_n_tokens, H = hidden_pl, grid = vit_image_size / vit_patch_size; - n_img_tokens = (int64_t) in.n_images * K; - img_emb_host.assign((size_t) in.n_images * K * H, 0.0f); + const int64_t K = vit_n_tokens, H = hidden_pl, grid = vit_image_size/vit_patch_size; + n_img_tokens = (int64_t) in.n_images*K; + img_emb_host.assign((size_t) in.n_images*K * H, 0.0f); - ggml_context * VC = vision_scratch.reset((size_t) 128 * 1024 * 1024); + ggml_context * VC = vision_scratch.reset((size_t) 128*1024*1024); if (!VC) { std::fprintf(stderr, "vla(pi05): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, vit_image_size, vit_image_size, 3); ggml_set_input(t_px); ggml_tensor * conv = ggml_conv_2d(VC, vit.patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); ggml_tensor * h = ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos); for (int64_t i = 0; i < vit_layers; ++i) - h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden / vit_heads, vit_hidden, vit_ln_eps); + h = build_siglip_layer(VC, vit.enc.blk[i], h, K, vit_heads, vit_hidden/vit_heads, vit_hidden, vit_ln_eps); h = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, h, vit_ln_eps), vit.post_ln_w), vit.post_ln_b); // PaliGemma projector: linear (+ optional bias), then 1/sqrt(hidden) scale (matches clip.cpp siglip.cpp). ggml_tensor * proj = ggml_mul_mat(VC, mm_proj_w, h); if (mm_proj_b) proj = ggml_add(VC, proj, mm_proj_b); - ggml_tensor * vit_emb = ggml_scale(VC, proj, 1.0f / std::sqrt((float) proj->ne[0])); + ggml_tensor * vit_emb = ggml_scale(VC, proj, 1.0f/std::sqrt((float) proj->ne[0])); ggml_set_output(vit_emb); ggml_cgraph * vg = ggml_new_graph_custom(VC, 8192, false); @@ -604,9 +604,9 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(pi05): vision compute failed (view %d)\n", v); return {}; } - ggml_backend_tensor_get(vit_emb, img_emb_host.data() + (size_t) v * K * H, 0, ggml_nbytes(vit_emb)); + ggml_backend_tensor_get(vit_emb, img_emb_host.data()+(size_t) v * K * H, 0, ggml_nbytes(vit_emb)); } - stats.ms_vision = std::chrono::duration(clk::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(clk::now()-tv0).count(); // Undo the 1/sqrt(hidden) the shared vision graph applies; pi05 wants raw // projector features. Inside this branch on purpose: precomputed_img_emb @@ -621,9 +621,9 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { return {}; } const int64_t n_lang = in.n_lang; - const int64_t n_prefix = n_img_tokens + n_lang; + const int64_t n_prefix = n_img_tokens+n_lang; - std::vector lang_ids(in.lang_tokens, in.lang_tokens + n_lang); + std::vector lang_ids(in.lang_tokens, in.lang_tokens+n_lang); std::vector lang_rows((size_t) n_lang * hidden_pl); { if (!io.fetch_rows_f32("token_embd.weight", lang_ids, lang_rows.data(), hidden_pl)) return {}; @@ -631,7 +631,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { // Prefix + expert graph depends only on the token counts and step count. const MainKey mkey{ n_img_tokens, n_lang, num_steps }; - const bool built = main_graph.ensure(backend, mkey, (size_t) 64 * 1024 * 1024, + const bool built = main_graph.ensure(backend, mkey, (size_t) 64*1024*1024, [&](ggml_context * C, MainIO & gio) -> ggml_cgraph * { ggml_tensor * t_image_emb = ggml_new_tensor_2d(C, GGML_TYPE_F32, hidden_pl, n_img_tokens); ggml_set_input(t_image_emb); ggml_tensor * t_lang_emb = ggml_new_tensor_2d(C, GGML_TYPE_F32, hidden_pl, n_lang); ggml_set_input(t_lang_emb); @@ -698,13 +698,13 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { { std::vector pp(n_prefix); for (int64_t i = 0; i < n_prefix; ++i) pp[i] = (int32_t) i; ggml_backend_tensor_set(t_prefix_pos, pp.data(), 0, ggml_nbytes(t_prefix_pos)); - std::vector sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix + i); + std::vector sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix+i); ggml_backend_tensor_set(t_suffix_pos, sp.data(), 0, ggml_nbytes(t_suffix_pos)); } { std::vector x0h((size_t) max_ad * chunk); if (in.noise) - std::memcpy(x0h.data(), in.noise, x0h.size() * sizeof(float)); + std::memcpy(x0h.data(), in.noise, x0h.size()*sizeof(float)); else { std::normal_distribution nd(0.f, 1.f); for (auto & v : x0h) @@ -713,37 +713,37 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_x0, x0h.data(), 0, ggml_nbytes(t_x0)); } for (int s = 0; s < num_steps; ++s) { - const float timestep = 1.0f + (float) s * dt; + const float timestep = 1.0f+(float) s * dt; const std::vector tv = sinusoidal_time_emb(timestep, hidden_ex, cfg.min_period, cfg.max_period); ggml_backend_tensor_set(t_time[s], tv.data(), 0, ggml_nbytes(t_time[s])); } const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); - stats.ms_inference = std::chrono::duration(clk::now() - ti0).count(); + stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); if (st != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(pi05): ggml_backend_graph_compute failed (%d)\n", (int) st); return {}; } std::vector out((size_t) chunk * max_ad); - ggml_backend_tensor_get(x_final, out.data(), 0, out.size() * sizeof(float)); + ggml_backend_tensor_get(x_final, out.data(), 0, out.size()*sizeof(float)); if (!vla::env_flag("VLA_PI05_SKIP_UNNORM")) { for (int64_t t = 0; t < chunk; ++t) { - float * row = out.data() + (size_t) t * max_ad; + float * row = out.data()+(size_t) t * max_ad; for (int64_t j = 0; j < max_ad; ++j) { if (j >= cfg.real_action_dim) row[j] = 0.0f; else if (quantile_norm) - row[j] = (row[j] + 1.0f) * (action_q99[j] - action_q01[j]) * 0.5f + action_q01[j]; + row[j] = (row[j]+1.0f)*(action_q99[j]-action_q01[j])*0.5f+action_q01[j]; else - row[j] = row[j] * (action_std[j] + cfg.norm_eps) + action_mean[j]; + row[j] = row[j]*(action_std[j]+cfg.norm_eps)+action_mean[j]; } } } - stats.ms_total = std::chrono::duration(clk::now() - t0).count(); + stats.ms_total = std::chrono::duration(clk::now()-t0).count(); return out; } diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 857d843..bc9b37b 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -69,7 +69,7 @@ struct safetensors { file.read(reinterpret_cast(&header_size), sizeof(header_size)); std::string header_str(header_size, '\0'); file.read(header_str.data(), header_size); - data_blob_start = sizeof(uint64_t) + header_size; + data_blob_start = sizeof(uint64_t)+header_size; json j = json::parse(header_str); for (auto it = j.begin(); it != j.end(); ++it) { if (it.key() == "__metadata__") @@ -113,14 +113,14 @@ struct safetensors { } want *= (size_t) d; } - if (info.off_end < info.off_begin || info.off_end - info.off_begin != want) { + if (info.off_end < info.off_begin || info.off_end-info.off_begin != want) { std::fprintf(stderr, "vla: bad data_offsets for %s\n", name.c_str()); return false; } - const size_t bytes = info.off_end - info.off_begin; - file.seekg(data_blob_start + info.off_begin, std::ios::beg); + const size_t bytes = info.off_end-info.off_begin; + file.seekg(data_blob_start+info.off_begin, std::ios::beg); if (info.dtype == "BF16") { - std::vector tmp(bytes / sizeof(ggml_bf16_t)); + std::vector tmp(bytes/sizeof(ggml_bf16_t)); file.read(reinterpret_cast(tmp.data()), bytes); ggml_bf16_to_fp32_row(tmp.data(), dst, tmp.size()); } else { @@ -137,12 +137,12 @@ struct safetensors { return false; } const auto & info = it->second; - if ((info.off_end - info.off_begin) != expected_bytes || + if ((info.off_end-info.off_begin) != expected_bytes || info.dtype != expected_dtype) { std::fprintf(stderr, "vla: bad raw read for %s\n", name.c_str()); return false; } - file.seekg(data_blob_start + info.off_begin, std::ios::beg); + file.seekg(data_blob_start+info.off_begin, std::ios::beg); file.read(static_cast(dst), expected_bytes); return true; } @@ -188,7 +188,7 @@ struct gguf_source { if (nd_used > GGML_MAX_DIMS) return false; for (int d = 0; d < (int) pt_shape.size(); ++d) { - const int64_t expected = pt_shape[pt_shape.size() - 1 - d]; + const int64_t expected = pt_shape[pt_shape.size()-1-d]; if (t->ne[d] != expected) return false; } @@ -211,7 +211,7 @@ struct gguf_source { return false; } const int64_t id = gguf_find_tensor(gctx, name.c_str()); - const size_t offset = data_off + gguf_get_tensor_offset(gctx, id); + const size_t offset = data_off+gguf_get_tensor_offset(gctx, id); const size_t bytes = gguf_get_tensor_size(gctx, id); if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) { std::fprintf(stderr, "vla: fseek failed for %s\n", name.c_str()); @@ -221,7 +221,7 @@ struct gguf_source { if (std::fread(dst, 1, bytes, fp) != bytes) return false; } else if (t->type == GGML_TYPE_BF16) { - std::vector tmp(bytes / sizeof(ggml_bf16_t)); + std::vector tmp(bytes/sizeof(ggml_bf16_t)); if (std::fread(tmp.data(), 1, bytes, fp) != bytes) return false; ggml_bf16_to_fp32_row(tmp.data(), dst, tmp.size()); @@ -248,7 +248,7 @@ struct gguf_source { std::fprintf(stderr, "vla: gguf bad raw read for %s\n", name.c_str()); return false; } - const size_t offset = data_off + gguf_get_tensor_offset(gctx, id); + const size_t offset = data_off+gguf_get_tensor_offset(gctx, id); if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) return false; return std::fread(dst, 1, bytes, fp) == bytes; @@ -393,7 +393,7 @@ namespace { // in-tree models. Bidirectional attention, F32 score accumulation, tanh GELU. ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_tensor * x, int64_t seq, int64_t heads, int64_t head_dim, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) head_dim); + const float scale = 1.0f/std::sqrt((float) head_dim); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); ggml_tensor * q = ggml_add(C, ggml_mul_mat(C, w.Wq, n1), w.bq); ggml_tensor * k = ggml_add(C, ggml_mul_mat(C, w.Wk, n1), w.bk); @@ -464,7 +464,7 @@ bool load_config_from_json(const std::string & path, Config & cfg) { cfg.n_layers = j.at("num_vlm_layers").get(); const double mul = j.at("expert_width_multiplier").get(); - cfg.expert_h = static_cast(std::round(double(cfg.hidden) * mul)); + cfg.expert_h = static_cast(std::round(double(cfg.hidden)*mul)); cfg.real_state_dim = j.at("input_features").at("observation.state").at("shape").at(0).get(); cfg.real_action_dim = j.at("output_features").at("action").at("shape").at(0).get(); @@ -475,7 +475,7 @@ bool load_config_from_json(const std::string & path, Config & cfg) { cfg.n_state = 1; cfg.q_full_dim = cfg.n_q_heads * cfg.head_dim; - cfg.kv_full_dim = cfg.n_kv_heads * cfg.head_dim; + cfg.kv_full_dim = cfg.n_kv_heads*cfg.head_dim; cfg.rope_n_dims = static_cast(cfg.head_dim); cfg.norm_eps = 1e-8f; @@ -564,7 +564,7 @@ std::string default_config_path(const std::string & ckpt_path) { bool ends_with_gguf(const std::string & path) { static const std::string sfx = ".gguf"; return path.size() >= sfx.size() - && path.compare(path.size() - sfx.size(), sfx.size(), sfx) == 0; + && path.compare(path.size()-sfx.size(), sfx.size(), sfx) == 0; } bool load_config_from_gguf(const gguf_source & st, Config & cfg) { @@ -626,7 +626,7 @@ bool load_config_from_gguf(const gguf_source & st, Config & cfg) { cfg.n_state = 1; cfg.q_full_dim = cfg.n_q_heads * cfg.head_dim; - cfg.kv_full_dim = cfg.n_kv_heads * cfg.head_dim; + cfg.kv_full_dim = cfg.n_kv_heads*cfg.head_dim; cfg.rope_n_dims = static_cast(cfg.head_dim); cfg.norm_eps = st.has_key("smolvla.norm_eps") ? st.get_f32("smolvla.norm_eps") : 1e-8f; @@ -690,8 +690,8 @@ std::string hf_to_gguf(const std::string & n) { const size_t end_i = rest.find('.', 7); if (end_i == std::string::npos) return n; - const std::string idx = rest.substr(7, end_i - 7); - const std::string suf = rest.substr(end_i + 1); + const std::string idx = rest.substr(7, end_i-7); + const std::string suf = rest.substr(end_i+1); return std::string(dst_blk) + ".blk." + idx + "." + map_suffix(suf); }; @@ -721,8 +721,8 @@ std::string hf_to_gguf(const std::string & n) { const size_t e = rest.find('.', 15); if (e == std::string::npos) return n; - const std::string idx = rest.substr(15, e - 15); - const std::string suf = rest.substr(e + 1); + const std::string idx = rest.substr(15, e-15); + const std::string suf = rest.substr(e+1); std::string ds; if (suf == "layer_norm1.weight") ds = "ln1.weight"; @@ -796,7 +796,7 @@ ggml_tensor * build_vlm_layer(ggml_context * ctx, const VlmLayerW & w, ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); ggml_tensor * K = ggml_permute(ctx, k_rope, 0, 2, 1, 3); ggml_tensor * V = ggml_permute(ctx, v_h, 0, 2, 1, 3); - const float scale = 1.f / std::sqrt(static_cast(cfg.head_dim)); + const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, K, V, mask, scale, 0.f, 0.f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); @@ -835,7 +835,7 @@ ggml_tensor * build_expert_self_attn_layer( ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); ggml_tensor * Kp = ggml_permute(ctx, K_full, 0, 2, 1, 3); ggml_tensor * Vp = ggml_permute(ctx, V_full, 0, 2, 1, 3); - const float scale = 1.f / std::sqrt(static_cast(cfg.head_dim)); + const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, Kp, Vp, mask_full, scale, 0.f, 0.f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); @@ -874,7 +874,7 @@ ggml_tensor * build_expert_cross_attn_layer( ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); ggml_tensor * Kp = ggml_permute(ctx, K_repro, 0, 2, 1, 3); ggml_tensor * Vp = ggml_permute(ctx, V_repro, 0, 2, 1, 3); - const float scale = 1.f / std::sqrt(static_cast(cfg.head_dim)); + const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, Kp, Vp, mask_prefix_only, scale, 0.f, 0.f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); @@ -899,15 +899,15 @@ static void vram_probe(ggml_backend_t backend, const char * label) { ggml_backend_dev_memory(dev, &free_b, &total_b); static size_t prev_free = 0; static bool have_prev = false; - const double MiB = 1024.0 * 1024.0; - const long long used = (long long)(total_b - free_b); + const double MiB = 1024.0*1024.0; + const long long used = (long long)(total_b-free_b); if (have_prev) { - const long long delta = (long long)prev_free - (long long)free_b; + const long long delta = (long long)prev_free-(long long)free_b; std::printf("vla: [vram] %-22s used=%.1f MiB free=%.1f MiB (+%.1f MiB)\n", - label, used / MiB, free_b / MiB, delta / MiB); + label, used/MiB, free_b/MiB, delta/MiB); } else { std::printf("vla: [vram] %-22s used=%.1f MiB free=%.1f MiB\n", - label, used / MiB, free_b / MiB); + label, used/MiB, free_b/MiB); } prev_free = free_b; have_prev = true; @@ -1005,8 +1005,8 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, m->vit_ln_eps = gst.get_f32("smolvla.vit_ln_eps"); } { - const int64_t grid = m->vit_image / m->vit_patch; - const int64_t k = grid / m->vit_scale; + const int64_t grid = m->vit_image/m->vit_patch; + const int64_t k = grid/m->vit_scale; if (k * k != m->vit_n_tokens) { std::fprintf(stderr, "vla: smolvla vit geometry mismatch (grid=%lld scale=%lld -> %lld tokens, KV says %lld)\n", (long long) grid, (long long) m->vit_scale, (long long) (k * k), (long long) m->vit_n_tokens); @@ -1015,8 +1015,8 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, } m->cfg.n_img = m->vit_n_tokens; } - m->cfg.n_prefix = m->cfg.n_img + m->cfg.n_lang + m->cfg.n_state; - m->cfg.n_full = m->cfg.n_prefix + m->cfg.n_suffix; + m->cfg.n_prefix = m->cfg.n_img+m->cfg.n_lang+m->cfg.n_state; + m->cfg.n_full = m->cfg.n_prefix+m->cfg.n_suffix; if (!use_gguf) { if (!st.open(ckpt_path)) { @@ -1029,7 +1029,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, int max_layer = -1; for (const auto & kv : st.tensors) { if (kv.first.compare(0, prefix.size(), prefix) == 0) { - const int idx = std::atoi(kv.first.c_str() + prefix.size()); + const int idx = std::atoi(kv.first.c_str()+prefix.size()); if (idx > max_layer) max_layer = idx; } @@ -1039,7 +1039,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, delete m; return nullptr; } - m->cfg.n_layers = max_layer + 1; + m->cfg.n_layers = max_layer+1; } { const auto it = st.tensors.find("model.vlm_with_expert.lm_expert.layers.0.mlp.gate_proj.weight"); @@ -1086,7 +1086,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, } ggml_init_params gparams = { - size_t(32) * 1024 * 1024, + size_t(32)*1024*1024, nullptr, true, }; @@ -1118,8 +1118,8 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, // Vision tower weights (SigLIP-B/16 encoder + single-linear pixel-shuffle connector). { const int64_t H = m->vit_hidden, FF = m->vit_inter, P = m->vit_patch; - const int64_t grid = m->vit_image / P, n_patches = grid * grid; - const int64_t c4 = H * m->vit_scale * m->vit_scale; + const int64_t grid = m->vit_image/P, n_patches = grid * grid; + const int64_t c4 = H * m->vit_scale*m->vit_scale; const char * VP = "model.vlm_with_expert.vlm.model.vision_model."; m->vit_patch_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, P, P, 3, H); m->vit_patch_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H); @@ -1191,7 +1191,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, m->expert_layers.resize(cfg.n_layers); for (int i = 0; i < cfg.n_layers; ++i) { ExpertLayerW & w = m->expert_layers[i]; - w.is_self_attn = (i % cfg.self_attn_every_n == 0); + w.is_self_attn = (i%cfg.self_attn_every_n == 0); w.Wln_in = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.expert_h); w.Wq = ggml_new_tensor_2d(ctx, wdt, cfg.expert_h, cfg.q_full_dim); if (w.is_self_attn) { @@ -1231,13 +1231,13 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, m->W_ain = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.max_action_dim, cfg.expert_h); m->b_ain = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.expert_h); - m->W_at1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2 * cfg.expert_h, cfg.expert_h); + m->W_at1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2*cfg.expert_h, cfg.expert_h); m->b_at1 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.expert_h); m->W_at2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.expert_h, cfg.expert_h); m->b_at2 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.expert_h); pending_f32.push_back({"model.action_in_proj.weight", m->W_ain, {cfg.expert_h, cfg.max_action_dim}}); pending_f32.push_back({"model.action_in_proj.bias", m->b_ain, {cfg.expert_h}}); - pending_f32.push_back({"model.action_time_mlp_in.weight", m->W_at1, {cfg.expert_h, 2 * cfg.expert_h}}); + pending_f32.push_back({"model.action_time_mlp_in.weight", m->W_at1, {cfg.expert_h, 2*cfg.expert_h}}); pending_f32.push_back({"model.action_time_mlp_in.bias", m->b_at1, {cfg.expert_h}}); pending_f32.push_back({"model.action_time_mlp_out.weight", m->W_at2, {cfg.expert_h, cfg.expert_h}}); pending_f32.push_back({"model.action_time_mlp_out.bias", m->b_at2, {cfg.expert_h}}); @@ -1260,7 +1260,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, return nullptr; } std::printf("vla: [vram] weight_buf = %.1f MiB\n", - ggml_backend_buffer_get_size(m->weight_buf) / (1024.0 * 1024.0)); + ggml_backend_buffer_get_size(m->weight_buf)/(1024.0*1024.0)); vram_probe(m->backend, "after weights alloc"); auto stream_f32 = [&](const std::string & hf_name, ggml_tensor * t, @@ -1303,18 +1303,18 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, } { - const float dt = -1.f / static_cast(cfg.num_steps); + const float dt = -1.f/static_cast(cfg.num_steps); for (int step = 0; step < cfg.num_steps; ++step) { - const double time = 1.0 + double(step) * double(dt); + const double time = 1.0+double(step)*double(dt); const auto te = sinusoidal_time_emb(time, cfg.expert_h, cfg.min_period, cfg.max_period); - std::vector tile(cfg.expert_h * cfg.n_suffix); + std::vector tile(cfg.expert_h*cfg.n_suffix); for (int64_t t = 0; t < cfg.n_suffix; ++t) { - std::memcpy(tile.data() + t * cfg.expert_h, - te.data(), cfg.expert_h * sizeof(float)); + std::memcpy(tile.data()+t * cfg.expert_h, + te.data(), cfg.expert_h*sizeof(float)); } ggml_backend_tensor_set(m->time_bcasts[step], tile.data(), - 0, tile.size() * sizeof(float)); + 0, tile.size()*sizeof(float)); } } @@ -1331,10 +1331,10 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { const Config & cfg_model = m->cfg; Config cfg = cfg_model; - cfg.n_img = cfg_model.n_img * int64_t(n_views); + cfg.n_img = cfg_model.n_img*int64_t(n_views); ggml_init_params gparams = { - size_t(64) * 1024 * 1024, + size_t(64)*1024*1024, nullptr, true, }; @@ -1345,8 +1345,8 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { } const int64_t n_lang_max = cfg.n_lang; - const int64_t n_prefix_max = cfg.n_img + n_lang_max + cfg.n_state; - const int64_t n_full_max = n_prefix_max + cfg.n_suffix; + const int64_t n_prefix_max = cfg.n_img+n_lang_max+cfg.n_state; + const int64_t n_full_max = n_prefix_max+cfg.n_suffix; ggml_tensor * img_emb_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.hidden, cfg.n_img); ggml_tensor * lang_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_lang_max); @@ -1402,7 +1402,7 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { cfg_built, &xk_cache[li], &xv_cache[li]); } - const float dt = -1.f / static_cast(cfg.num_steps); + const float dt = -1.f/static_cast(cfg.num_steps); ggml_tensor * x_t = x0; for (int step = 0; step < cfg.num_steps; ++step) { @@ -1450,7 +1450,7 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { } std::printf("vla: [vram] gallocr compute buf = %.1f MiB\n", - ggml_gallocr_get_buffer_size(galloc, 0) / (1024.0 * 1024.0)); + ggml_gallocr_get_buffer_size(galloc, 0)/(1024.0*1024.0)); vram_probe(m->backend, "after gallocr reserve"); m->ctx_compute = ctx; @@ -1495,7 +1495,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { Config cfg = m->cfg; - const size_t per_view_n = size_t(m->cfg.n_img * cfg.hidden); + const size_t per_view_n = size_t(m->cfg.n_img*cfg.hidden); int n_views = 0; size_t img_emb_n = 0; @@ -1509,7 +1509,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } n_views = in.n_img_views; img_emb_n = per_view_n * size_t(n_views); - img_emb_pre.assign(in.precomputed_img_emb, in.precomputed_img_emb + img_emb_n); + img_emb_pre.assign(in.precomputed_img_emb, in.precomputed_img_emb+img_emb_n); } else { if (in.n_images < 1 || in.images == nullptr) { @@ -1520,19 +1520,19 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { img_emb_n = per_view_n * size_t(n_views); img_emb_pre.resize(img_emb_n); - const int64_t H = m->vit_hidden, grid = m->vit_image / m->vit_patch, n_patches = grid * grid; + const int64_t H = m->vit_hidden, grid = m->vit_image/m->vit_patch, n_patches = grid * grid; const int64_t s = m->vit_scale, c4 = H * s * s, K = m->vit_n_tokens; const auto t_vision_begin = clock::now(); // Graph A: SigLIP ViT (conv patch-embed -> +pos -> layers -> post_ln), plain sequential positions. - ggml_context * VC = m->vision_scratch.reset(size_t(256) * 1024 * 1024); + ggml_context * VC = m->vision_scratch.reset(size_t(256)*1024*1024); if (!VC) { std::fprintf(stderr, "vla(smolvla): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_px = ggml_new_tensor_3d(VC, GGML_TYPE_F32, m->vit_image, m->vit_image, 3); ggml_set_input(t_px); ggml_tensor * conv = ggml_conv_2d(VC, m->vit_patch_w, t_px, (int) m->vit_patch, (int) m->vit_patch, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, n_patches, H))); ggml_tensor * hv = ggml_add(VC, ggml_add(VC, patches, m->vit_patch_b), m->vit_pos); for (int64_t i = 0; i < m->vit_layers; ++i) - hv = build_siglip_layer(VC, m->vit[i], hv, n_patches, m->vit_heads, H / m->vit_heads, H, m->vit_ln_eps); + hv = build_siglip_layer(VC, m->vit[i], hv, n_patches, m->vit_heads, H/m->vit_heads, H, m->vit_ln_eps); ggml_tensor * post_ln = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, hv, m->vit_ln_eps), m->vit_post_ln_w), m->vit_post_ln_b); ggml_set_output(post_ln); ggml_cgraph * gA = ggml_new_graph_custom(VC, 8192, false); @@ -1543,7 +1543,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } // Graph B: pixel-shuffle connector, a single bias-free matmul (c4 -> hidden). - ggml_context * MC = m->connector_scratch.reset(size_t(64) * 1024 * 1024); + ggml_context * MC = m->connector_scratch.reset(size_t(64)*1024*1024); if (!MC) { std::fprintf(stderr, "vla(smolvla): ggml_init(connector ctx) failed\n"); return {}; } ggml_tensor * t_shuf = ggml_new_tensor_2d(MC, GGML_TYPE_F32, c4, K); ggml_set_input(t_shuf); ggml_tensor * img_embeds = ggml_mul_mat(MC, m->mm_fc, t_shuf); @@ -1555,7 +1555,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { return {}; } - std::vector chw, post_host((size_t) H * n_patches), shuf_host((size_t) c4 * K); + std::vector chw, post_host((size_t) H * n_patches), shuf_host((size_t) c4*K); bool vok = true; for (int v = 0; v < n_views && vok; ++v) { if (!preprocess_image_chw("smolvla", in.images[v], m->vit_image, chw)) { @@ -1572,15 +1572,15 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { if (ggml_backend_graph_compute(m->backend, gB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): connector compute failed (view %d)\n", v); vok = false; break; } - ggml_backend_tensor_get(img_embeds, img_emb_pre.data() + size_t(v) * per_view_n, 0, ggml_nbytes(img_embeds)); + ggml_backend_tensor_get(img_embeds, img_emb_pre.data()+size_t(v)*per_view_n, 0, ggml_nbytes(img_embeds)); } if (!vok) return {}; - m->stats.ms_vision = std::chrono::duration(clock::now() - t_vision_begin).count(); + m->stats.ms_vision = std::chrono::duration(clock::now()-t_vision_begin).count(); } - cfg.n_img = m->cfg.n_img * int64_t(n_views); - cfg.n_prefix = cfg.n_img + cfg.n_lang + cfg.n_state; - cfg.n_full = cfg.n_prefix + cfg.n_suffix; + cfg.n_img = m->cfg.n_img*int64_t(n_views); + cfg.n_prefix = cfg.n_img+cfg.n_lang+cfg.n_state; + cfg.n_full = cfg.n_prefix+cfg.n_suffix; if (in.n_lang < 1 || in.n_lang > int(cfg.n_lang)) { std::fprintf(stderr, "vla: lang_tokens length %d out of range [1, %lld]\n", @@ -1618,21 +1618,21 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } const int64_t n_lang_max = m->cfg.n_lang; - const int64_t n_prefix_max = cfg.n_img + n_lang_max + cfg.n_state; - const int64_t n_full_max = n_prefix_max + cfg.n_suffix; - const int64_t pad_start = cfg.n_img + in.n_lang; - const int64_t pad_end = cfg.n_img + n_lang_max; + const int64_t n_prefix_max = cfg.n_img+n_lang_max+cfg.n_state; + const int64_t n_full_max = n_prefix_max+cfg.n_suffix; + const int64_t pad_start = cfg.n_img+in.n_lang; + const int64_t pad_end = cfg.n_img+n_lang_max; std::vector state_host(cfg.max_state_dim, 0.0f); if (in.state) - std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); + std::memcpy(state_host.data(), in.state, cfg.max_state_dim*sizeof(float)); for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { - state_host[i] = (state_host[i] - m->state_mean[i]) / (m->state_std[i] + cfg.norm_eps); + state_host[i] = (state_host[i]-m->state_mean[i])/(m->state_std[i]+cfg.norm_eps); } - std::vector noise_host(cfg.n_suffix * cfg.max_action_dim); + std::vector noise_host(cfg.n_suffix*cfg.max_action_dim); if (in.noise) { - std::memcpy(noise_host.data(), in.noise, noise_host.size() * sizeof(float)); + std::memcpy(noise_host.data(), in.noise, noise_host.size()*sizeof(float)); } else { std::normal_distribution dist(0.f, 1.f); for (auto & v : noise_host) @@ -1640,22 +1640,22 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } std::vector lang_host(n_lang_max, 0); - std::memcpy(lang_host.data(), in.lang_tokens, in.n_lang * sizeof(int32_t)); + std::memcpy(lang_host.data(), in.lang_tokens, in.n_lang*sizeof(int32_t)); - const int64_t state_pos = cfg.n_img + in.n_lang; - const int64_t suffix_pos_base = state_pos + 1; + const int64_t state_pos = cfg.n_img+in.n_lang; + const int64_t suffix_pos_base = state_pos+1; std::vector mask_prefill_host(n_prefix_max * n_prefix_max); std::vector pos_prefill_host (n_prefix_max); for (int64_t i = 0; i < n_prefix_max; ++i) { for (int64_t j = 0; j < n_prefix_max; ++j) { bool blocked = false; - if ((i < n_prefix_max - 1) && (j == n_prefix_max - 1)) + if ((i < n_prefix_max-1) && (j == n_prefix_max-1)) blocked = true; if (j >= pad_start && j < pad_end) blocked = true; - mask_prefill_host[i * n_prefix_max + j] = blocked ? -INFINITY : 0.f; + mask_prefill_host[i * n_prefix_max+j] = blocked ? -INFINITY : 0.f; } - pos_prefill_host[i] = (i == n_prefix_max - 1) + pos_prefill_host[i] = (i == n_prefix_max-1) ? static_cast(state_pos) : static_cast(i); } @@ -1670,16 +1670,16 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { if (j < n_prefix_max) { blocked = (j >= pad_start && j < pad_end); } else { - blocked = ((j - n_prefix_max) > i); + blocked = ((j-n_prefix_max) > i); } - mask_full_host[i * n_full_max + j] = blocked ? -INFINITY : 0.f; + mask_full_host[i * n_full_max+j] = blocked ? -INFINITY : 0.f; } for (int64_t j = 0; j < n_prefix_max; ++j) { if (j >= pad_start && j < pad_end) { - mask_prefix_only_host[i * n_prefix_max + j] = -INFINITY; + mask_prefix_only_host[i * n_prefix_max+j] = -INFINITY; } } - pos_full_host [i] = static_cast(suffix_pos_base + i); + pos_full_host [i] = static_cast(suffix_pos_base+i); pos_rebased_host[i] = static_cast(i); } @@ -1690,12 +1690,12 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_backend_tensor_set(m->in_img_emb, img_emb_pre.data(), 0, img_emb_n * sizeof(float)); ggml_backend_tensor_set(m->in_lang_ids, lang_host.data(), 0, n_lang_max * sizeof(int32_t)); - ggml_backend_tensor_set(m->in_state, state_host.data(), 0, cfg.max_state_dim * sizeof(float)); - ggml_backend_tensor_set(m->in_x0, noise_host.data(), 0, noise_host.size() * sizeof(float)); + ggml_backend_tensor_set(m->in_state, state_host.data(), 0, cfg.max_state_dim*sizeof(float)); + ggml_backend_tensor_set(m->in_x0, noise_host.data(), 0, noise_host.size()*sizeof(float)); ggml_backend_tensor_set(m->in_mask_prefill, mask_prefill_host.data(), 0, mask_prefill_host.size() * sizeof(float)); ggml_backend_tensor_set(m->in_pos_prefill, pos_prefill_host.data(), 0, pos_prefill_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(m->in_mask_full, mask_full_host.data(), 0, mask_full_host.size() * sizeof(float)); - ggml_backend_tensor_set(m->in_mask_pfx_only, mask_prefix_only_host.data(), 0, mask_prefix_only_host.size() * sizeof(float)); + ggml_backend_tensor_set(m->in_mask_pfx_only, mask_prefix_only_host.data(), 0, mask_prefix_only_host.size()*sizeof(float)); ggml_backend_tensor_set(m->in_pos_full, pos_full_host.data(), 0, pos_full_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(m->in_pos_rebased, pos_rebased_host.data(), 0, pos_rebased_host.size() * sizeof(int32_t)); @@ -1705,24 +1705,24 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { return {}; } m->stats.ms_inference = std::chrono::duration( - clock::now() - t0).count(); + clock::now()-t0).count(); - std::vector out(cfg.n_suffix * cfg.max_action_dim); - ggml_backend_tensor_get(m->out_x_t, out.data(), 0, out.size() * sizeof(float)); + std::vector out(cfg.n_suffix*cfg.max_action_dim); + ggml_backend_tensor_get(m->out_x_t, out.data(), 0, out.size()*sizeof(float)); for (int64_t r = 0; r < cfg.n_suffix; ++r) { - float * row = out.data() + r * cfg.max_action_dim; + float * row = out.data()+r * cfg.max_action_dim; for (int64_t j = 0; j < cfg.max_action_dim; ++j) { - row[j] = j < cfg.real_action_dim ? row[j] * (m->action_std[j] + cfg.norm_eps) + m->action_mean[j] : 0.0f; + row[j] = j < cfg.real_action_dim ? row[j]*(m->action_std[j]+cfg.norm_eps)+m->action_mean[j] : 0.0f; } } m->stats.ms_total = std::chrono::duration( - clock::now() - t_total_begin).count(); + clock::now()-t_total_begin).count(); return out; } ggml_init_params gparams = { - size_t(64) * 1024 * 1024, + size_t(64)*1024*1024, nullptr, true, }; @@ -1740,8 +1740,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } cfg.n_lang = in.n_lang; - cfg.n_prefix = cfg.n_img + cfg.n_lang + cfg.n_state; - cfg.n_full = cfg.n_prefix + cfg.n_suffix; + cfg.n_prefix = cfg.n_img+cfg.n_lang+cfg.n_state; + cfg.n_full = cfg.n_prefix+cfg.n_suffix; ggml_tensor * img_emb_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.hidden, cfg.n_img); ggml_tensor * lang_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, cfg.n_lang); ggml_tensor * state_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.max_state_dim); @@ -1756,32 +1756,32 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector state_host(cfg.max_state_dim, 0.0f); if (in.state) - std::memcpy(state_host.data(), in.state, cfg.max_state_dim * sizeof(float)); + std::memcpy(state_host.data(), in.state, cfg.max_state_dim*sizeof(float)); for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { - state_host[i] = (state_host[i] - m->state_mean[i]) / (m->state_std[i] + cfg.norm_eps); + state_host[i] = (state_host[i]-m->state_mean[i])/(m->state_std[i]+cfg.norm_eps); } - std::vector noise_host(cfg.n_suffix * cfg.max_action_dim); + std::vector noise_host(cfg.n_suffix*cfg.max_action_dim); if (in.noise) { - std::memcpy(noise_host.data(), in.noise, noise_host.size() * sizeof(float)); + std::memcpy(noise_host.data(), in.noise, noise_host.size()*sizeof(float)); } else { std::normal_distribution dist(0.f, 1.f); for (auto & v : noise_host) v = dist(m->rng); } - std::vector mask_prefill_host(cfg.n_prefix * cfg.n_prefix); + std::vector mask_prefill_host(cfg.n_prefix*cfg.n_prefix); std::vector pos_prefill_host (cfg.n_prefix); for (int64_t i = 0; i < cfg.n_prefix; ++i) { for (int64_t j = 0; j < cfg.n_prefix; ++j) { - const bool blocked = (i < cfg.n_prefix - 1) && (j == cfg.n_prefix - 1); - mask_prefill_host[i * cfg.n_prefix + j] = blocked ? -INFINITY : 0.f; + const bool blocked = (i < cfg.n_prefix-1) && (j == cfg.n_prefix-1); + mask_prefill_host[i * cfg.n_prefix+j] = blocked ? -INFINITY : 0.f; } pos_prefill_host[i] = static_cast(i); } std::vector mask_full_host (cfg.n_full * cfg.n_suffix); - std::vector mask_prefix_only_host(cfg.n_prefix * cfg.n_suffix, 0.f); + std::vector mask_prefix_only_host(cfg.n_prefix*cfg.n_suffix, 0.f); std::vector pos_full_host (cfg.n_suffix); std::vector pos_rebased_host (cfg.n_suffix); for (int64_t i = 0; i < cfg.n_suffix; ++i) { @@ -1790,10 +1790,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { if (j < cfg.n_prefix) blocked = false; else - blocked = ((j - cfg.n_prefix) > i); - mask_full_host[i * cfg.n_full + j] = blocked ? -INFINITY : 0.f; + blocked = ((j-cfg.n_prefix) > i); + mask_full_host[i * cfg.n_full+j] = blocked ? -INFINITY : 0.f; } - pos_full_host [i] = static_cast(cfg.n_prefix + i); + pos_full_host [i] = static_cast(cfg.n_prefix+i); pos_rebased_host[i] = static_cast(i); } @@ -1831,13 +1831,13 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } auto ms_since = [](auto t0) { - return std::chrono::duration(clock::now() - t0).count(); + return std::chrono::duration(clock::now()-t0).count(); }; auto & K_ref = (in.timing_detail == TimingDetail::PHASE) ? K_storage : k_cache; auto & V_ref = (in.timing_detail == TimingDetail::PHASE) ? V_storage : v_cache; - const float dt = -1.f / static_cast(cfg.num_steps); + const float dt = -1.f/static_cast(cfg.num_steps); ggml_tensor * x_t = x0; std::vector time_bcasts(cfg.num_steps, nullptr); std::vector> time_host (cfg.num_steps); @@ -1850,7 +1850,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } for (int step = 0; step < cfg.num_steps; ++step) { - const double time = 1.0 + static_cast(step) * static_cast(dt); + const double time = 1.0+static_cast(step)*static_cast(dt); time_host[step] = sinusoidal_time_emb(time, cfg.expert_h, cfg.min_period, cfg.max_period); ggml_tensor * time_bcast = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.expert_h, cfg.n_suffix); @@ -1889,22 +1889,22 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_backend_tensor_set(img_emb_in, img_emb_pre.data(), 0, img_emb_n * sizeof(float)); ggml_backend_tensor_set(lang_ids, in.lang_tokens, 0, cfg.n_lang * sizeof(int32_t)); - ggml_backend_tensor_set(state_t, state_host.data(), 0, cfg.max_state_dim * sizeof(float)); + ggml_backend_tensor_set(state_t, state_host.data(), 0, cfg.max_state_dim*sizeof(float)); ggml_backend_tensor_set(x0, noise_host.data(), 0, noise_host.size() * sizeof(float)); ggml_backend_tensor_set(mask_prefill, mask_prefill_host.data(), 0, mask_prefill_host.size() * sizeof(float)); ggml_backend_tensor_set(pos_prefill, pos_prefill_host.data(), 0, pos_prefill_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(mask_full, mask_full_host.data(), 0, mask_full_host.size() * sizeof(float)); - ggml_backend_tensor_set(mask_prefix_only, mask_prefix_only_host.data(), 0, mask_prefix_only_host.size() * sizeof(float)); + ggml_backend_tensor_set(mask_prefix_only, mask_prefix_only_host.data(), 0, mask_prefix_only_host.size()*sizeof(float)); ggml_backend_tensor_set(pos_full, pos_full_host.data(), 0, pos_full_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(pos_rebased, pos_rebased_host.data(), 0, pos_rebased_host.size() * sizeof(int32_t)); for (int step = 0; step < cfg.num_steps; ++step) { - std::vector tile(cfg.expert_h * cfg.n_suffix); + std::vector tile(cfg.expert_h*cfg.n_suffix); for (int64_t t = 0; t < cfg.n_suffix; ++t) { - std::memcpy(tile.data() + t * cfg.expert_h, - time_host[step].data(), cfg.expert_h * sizeof(float)); + std::memcpy(tile.data()+t * cfg.expert_h, + time_host[step].data(), cfg.expert_h*sizeof(float)); } - ggml_backend_tensor_set(time_bcasts[step], tile.data(), 0, tile.size() * sizeof(float)); + ggml_backend_tensor_set(time_bcasts[step], tile.data(), 0, tile.size()*sizeof(float)); } if (in.timing_detail == TimingDetail::PHASE) { @@ -1944,16 +1944,16 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { m->stats.ms_denoise = ms; } - m->stats.ms_inference = m->stats.ms_prefill + ms; + m->stats.ms_inference = m->stats.ms_prefill+ms; } - std::vector out(cfg.n_suffix * cfg.max_action_dim); - ggml_backend_tensor_get(x_t, out.data(), 0, out.size() * sizeof(float)); + std::vector out(cfg.n_suffix*cfg.max_action_dim); + ggml_backend_tensor_get(x_t, out.data(), 0, out.size()*sizeof(float)); for (int64_t r = 0; r < cfg.n_suffix; ++r) { - float * row = out.data() + r * cfg.max_action_dim; + float * row = out.data()+r * cfg.max_action_dim; for (int64_t j = 0; j < cfg.max_action_dim; ++j) { - row[j] = j < cfg.real_action_dim ? row[j] * (m->action_std[j] + cfg.norm_eps) + m->action_mean[j] : 0.0f; + row[j] = j < cfg.real_action_dim ? row[j]*(m->action_std[j]+cfg.norm_eps)+m->action_mean[j] : 0.0f; } } @@ -1961,7 +1961,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_free(ctx); m->stats.ms_total = std::chrono::duration( - clock::now() - t_total_begin).count(); + clock::now()-t_total_begin).count(); return out; } } diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 81d982c..aae2610 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -57,8 +57,8 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, } else { size_t b = js.find('{'); size_t q = js.find('"', b); - size_t qe = js.find('"', q + 1); - suite = js.substr(q + 1, qe - q - 1); suite_pos = q; + size_t qe = js.find('"', q+1); + suite = js.substr(q+1, qe-q-1); suite_pos = q; } if (suite_pos == std::string::npos) { std::fprintf(stderr, "vla(vla_adapter): suite '%s' not in stats\n", suite.c_str()); @@ -72,7 +72,7 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, size_t lb = js.find('[', k); size_t rb = js.find(']', lb); if (lb == std::string::npos || rb == std::string::npos) return false; - out.clear(); size_t p = lb + 1; + out.clear(); size_t p = lb+1; while (p < rb) { while (p < rb && (js[p] == ',' || js[p] == ' ' || js[p] == '\n' || js[p] == '\t' || js[p] == '\r')) ++p; @@ -84,7 +84,7 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, p += t ? 4 : 5; } else { - out.push_back(std::strtof(js.c_str() + p, nullptr)); + out.push_back(std::strtof(js.c_str()+p, nullptr)); while (p < rb && js[p] != ',') ++p; } @@ -360,9 +360,9 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(vla_adapter): stop_id %lld out of vocab\n", (long long) stop_id); return {}; } - const int64_t NUM_PROMPT_TOKENS = NPROMPT - 1; + const int64_t NUM_PROMPT_TOKENS = NPROMPT-1; const int64_t NPATCH = NP * n_views; - const int64_t SEQ = 1 + NPATCH + (NPROMPT-1) + num_tokens + 1; + const int64_t SEQ = 1+NPATCH+(NPROMPT-1)+num_tokens+1; const auto ti=clock::now(); // LM + action head graph depends only on the sequence layout. const MainKey mkey{ SEQ, n_views, NPROMPT }; diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 3d07ec5..be6dd00 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -151,8 +151,8 @@ bool load_config(const gguf_reader & g, VlaJepaModelArch & m, Config & cfg) { // merge_block_coords only enumerates the patch grid exactly when the spatial // merge divides it; otherwise it emits rows past the position table. - if (m.patch_size <= 0 || m.spatial_merge <= 0 || m.image_target_size % m.patch_size != 0 || - (m.image_target_size / m.patch_size) % m.spatial_merge != 0) { + if (m.patch_size <= 0 || m.spatial_merge <= 0 || m.image_target_size%m.patch_size != 0 || + (m.image_target_size/m.patch_size)%m.spatial_merge != 0) { std::fprintf(stderr, "vla(vla_jepa): image %lld / patch %lld / merge %lld do not divide evenly\n", (long long) m.image_target_size, (long long) m.patch_size, (long long) m.spatial_merge); return false; @@ -187,7 +187,7 @@ bool load_config(const gguf_reader & g, VlaJepaModelArch & m, Config & cfg) { m.dit.cfg.norm_out_eps = m.dit_norm_out_eps; cfg = Config{}; - cfg.n_img = (m.image_target_size / m.patch_size / m.spatial_merge) * (m.image_target_size / m.patch_size / m.spatial_merge); + cfg.n_img = (m.image_target_size/m.patch_size/m.spatial_merge)*(m.image_target_size/m.patch_size/m.spatial_merge); cfg.n_lang = 1024; cfg.n_state = 1; cfg.n_suffix = m.action_horizon; cfg.max_state_dim = m.state_dim; cfg.max_action_dim = m.action_dim; cfg.real_state_dim = m.state_dim; cfg.real_action_dim = m.action_dim; @@ -288,8 +288,8 @@ bool VlaJepaModelArch::build_caches() { if (caches_ready) return true; const int64_t side = image_target_size, ps = patch_size, m2 = spatial_merge; - const int64_t grid = side / ps; - const int64_t hd_vit = vit_hidden / vit_heads; + const int64_t grid = side/ps; + const int64_t hd_vit = vit_hidden/vit_heads; const int64_t num_side = (int64_t) std::lround(std::sqrt((double) vit_num_pos)); merge_block_coords(grid, grid, m2, c_grow, c_gcol); @@ -307,7 +307,7 @@ bool VlaJepaModelArch::build_caches() { c_tau.assign((size_t) num_steps, {}); c_tproj.assign((size_t) num_steps, {}); for (int64_t s = 0; s < num_steps; ++s) { - const int64_t bucket = (int64_t) ((double) s / (double) num_steps * (double) num_buckets); + const int64_t bucket = (int64_t) ((double) s/(double) num_steps * (double) num_buckets); action_sinusoid(bucket, dit_hidden, action_horizon, c_tau[(size_t) s]); timesteps_proj(bucket, c_tproj[(size_t) s]); } @@ -321,9 +321,9 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { const int64_t H = lm_hidden, E = dit_hidden, AD = action_dim, AH = action_horizon, OUTD = output_dim; const int64_t side = image_target_size, ps = patch_size, m2 = spatial_merge; - const int64_t grid = side / ps, n_patches = grid * grid, K = (grid / m2) * (grid / m2); - const int64_t hd_vit = vit_hidden / vit_heads; - const int64_t Nseq = 1 + num_future + AH; + const int64_t grid = side/ps, n_patches = grid * grid, K = (grid/m2)*(grid/m2); + const int64_t hd_vit = vit_hidden/vit_heads; + const int64_t Nseq = 1+num_future+AH; const char * dump_prefix = std::getenv("VLA_JEPA_DUMP"); if (!caches_ready) { std::fprintf(stderr, "vla(vla_jepa): caches not ready\n"); return {}; } @@ -331,8 +331,8 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { if (!dump_prefix) return; const int64_t n0 = t->ne[0], n1 = t->ne[1]; - std::vector buf((size_t) n0 * std::max(1, n1)); - ggml_backend_tensor_get(t, buf.data(), 0, buf.size() * sizeof(float)); + std::vector buf((size_t) n0*std::max(1, n1)); + ggml_backend_tensor_get(t, buf.data(), 0, buf.size()*sizeof(float)); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, name, (long long) n0, (long long) n1); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(buf.data(), sizeof(float), buf.size(), fp); @@ -342,7 +342,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector x_init((size_t) AH * AD); if (in.noise) - std::memcpy(x_init.data(), in.noise, x_init.size() * sizeof(float)); + std::memcpy(x_init.data(), in.noise, x_init.size()*sizeof(float)); else { std::mt19937 rng((uint32_t) std::chrono::steady_clock::now().time_since_epoch().count()); std::normal_distribution nd(0.f, 1.f); @@ -382,7 +382,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { } if (inj_patches.empty() && !in.images) { std::fprintf(stderr, "vla(vla_jepa): n_images=%d but the images pointer is null\n", in.n_images); return {}; } - ggml_context * VC = vision_scratch.reset((size_t) 512 * 1024 * 1024); + ggml_context * VC = vision_scratch.reset((size_t) 512*1024*1024); if (!VC) { std::fprintf(stderr, "vla(vla_jepa): ggml_init(vision ctx) failed\n"); return {}; } ggml_tensor * t_patches = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_patch_flat, n_patches); ggml_set_input(t_patches); ggml_tensor * t_pos = ggml_new_tensor_2d(VC, GGML_TYPE_F32, vit_hidden, n_patches); ggml_set_input(t_pos); @@ -415,7 +415,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { bool vok = true; for (int64_t v = 0; v < n_views && vok; ++v) { if (!inj_patches.empty()) { - ggml_backend_tensor_set(t_patches, inj_patches.data() + v * n_patches * vit_patch_flat, 0, ggml_nbytes(t_patches)); + ggml_backend_tensor_set(t_patches, inj_patches.data()+v * n_patches * vit_patch_flat, 0, ggml_nbytes(t_patches)); } else { if (!preprocess_image_patches("vla_jepa", in.images[v], side, ps, temporal_patch, c_grow, c_gcol, patches)) { vok = false; @@ -431,12 +431,12 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { vok = false; break; } - ggml_backend_tensor_get(vit_embeds, img_emb_host.data() + v * K * H, 0, ggml_nbytes(vit_embeds)); + ggml_backend_tensor_get(vit_embeds, img_emb_host.data()+v * K * H, 0, ggml_nbytes(vit_embeds)); for (int j = 0; j < 3; ++j) - ggml_backend_tensor_get(ds_out[j], ds_host[j].data() + v * K * H, 0, ggml_nbytes(ds_out[j])); - if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data() + v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } + ggml_backend_tensor_get(ds_out[j], ds_host[j].data()+v * K * H, 0, ggml_nbytes(ds_out[j])); + if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data()+v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } } - stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now() - tv0).count(); + stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); if (!vok) return {}; const int64_t n_img = n_views * K; @@ -446,9 +446,9 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { if (in.lang_tokens[j] == (int32_t) image_token_index) ++n_img_slots; if (n_img_slots == n_img) { - input_ids.assign(in.lang_tokens, in.lang_tokens + in.n_lang); + input_ids.assign(in.lang_tokens, in.lang_tokens+in.n_lang); } else if (n_img_slots == 0) { - input_ids.reserve(n_img + in.n_lang); + input_ids.reserve(n_img+in.n_lang); for (int64_t i = 0; i < n_img; ++i) input_ids.push_back((int32_t) image_token_index); for (int j = 0; j < in.n_lang; ++j) @@ -460,7 +460,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector inputs_embeds((size_t) SEQ * H); if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), H)) return {}; - { int64_t k = 0; for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { std::memcpy(inputs_embeds.data() + p * H, img_emb_host.data() + k * H, H * sizeof(float)); ++k; } } + { int64_t k = 0; for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { std::memcpy(inputs_embeds.data()+p * H, img_emb_host.data()+k * H, H * sizeof(float)); ++k; } } std::vector image_pos_idx, emb_pos_idx; for (int64_t p = 0; p < SEQ; ++p) { @@ -475,14 +475,14 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { for (int j = 0; j < 3; ++j) { ds_pad[j].assign((size_t) SEQ * H, 0.0f); for (int64_t k = 0; k < n_img; ++k) - std::memcpy(ds_pad[j].data() + (size_t) image_pos_idx[k] * H, ds_host[j].data() + (size_t) k * H, H * sizeof(float)); + std::memcpy(ds_pad[j].data()+(size_t) image_pos_idx[k]*H, ds_host[j].data()+(size_t) k * H, H * sizeof(float)); } const LmKey lkey{ SEQ, num_future }; - const bool lm_built = lm_graph.ensure(backend, lkey, (size_t) 512 * 1024 * 1024, + const bool lm_built = lm_graph.ensure(backend, lkey, (size_t) 512*1024*1024, [&](ggml_context * C, LmIO & gio) -> ggml_cgraph * { ggml_tensor * t_embeds = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_embeds); - ggml_tensor * t_pos2 = ggml_new_tensor_1d(C, GGML_TYPE_I32, 4 * SEQ); ggml_set_input(t_pos2); + ggml_tensor * t_pos2 = ggml_new_tensor_1d(C, GGML_TYPE_I32, 4*SEQ); ggml_set_input(t_pos2); ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); ggml_tensor * t_emb_idx= ggml_new_tensor_1d(C, GGML_TYPE_I32, num_future); ggml_set_input(t_emb_idx); ggml_tensor * t_ds[3]; @@ -520,8 +520,8 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); { - const int64_t llm_grid = side / ps / m2; - std::vector pp((size_t) 4 * SEQ, 0); + const int64_t llm_grid = side/ps/m2; + std::vector pp((size_t) 4*SEQ, 0); int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; @@ -530,9 +530,9 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { break; } const int64_t text_end = (img_start < 0) ? SEQ : img_start; - const int64_t text_len = text_end - st; + const int64_t text_len = text_end-st; for (int64_t i = 0; i < text_len; ++i) { - const int32_t p = (int32_t) (i + st_idx); + const int32_t p = (int32_t) (i+st_idx); pp[0*SEQ+(st+i)]=p; pp[1*SEQ+(st+i)]=p; pp[2*SEQ+(st+i)]=p; @@ -542,17 +542,17 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { break; } int64_t img_end = img_start; while (img_end < SEQ && input_ids[img_end] == (int32_t) image_token_index) ++img_end; - const int64_t n_img_tokens = img_end - img_start; - const int64_t this_t = n_img_tokens / (llm_grid * llm_grid); - const int64_t image_offset = text_len + st_idx; + const int64_t n_img_tokens = img_end-img_start; + const int64_t this_t = n_img_tokens/(llm_grid * llm_grid); + const int64_t image_offset = text_len+st_idx; for (int64_t tt = 0; tt < this_t; ++tt) for (int64_t hy = 0; hy < llm_grid; ++hy) for (int64_t wx = 0; wx < llm_grid; ++wx) { - const int64_t tok = img_start + (tt * llm_grid + hy) * llm_grid + wx; - pp[0*SEQ+tok] = (int32_t)(image_offset + tt); pp[1*SEQ+tok] = (int32_t)(image_offset + hy); pp[2*SEQ+tok] = (int32_t)(image_offset + wx); + const int64_t tok = img_start+(tt * llm_grid+hy)*llm_grid+wx; + pp[0*SEQ+tok] = (int32_t)(image_offset+tt); pp[1*SEQ+tok] = (int32_t)(image_offset+hy); pp[2*SEQ+tok] = (int32_t)(image_offset+wx); } - int64_t max_image_pos = this_t - 1; if (llm_grid - 1 > max_image_pos) max_image_pos = llm_grid - 1; - st_idx = image_offset + max_image_pos + 1; st = img_end; + int64_t max_image_pos = this_t-1; if (llm_grid-1 > max_image_pos) max_image_pos = llm_grid-1; + st_idx = image_offset+max_image_pos+1; st = img_end; } - std::memcpy(pp.data() + (size_t) 3 * SEQ, pp.data(), (size_t) SEQ * sizeof(int32_t)); + std::memcpy(pp.data()+(size_t) 3*SEQ, pp.data(), (size_t) SEQ * sizeof(int32_t)); ggml_backend_tensor_set(t_pos2, pp.data(), 0, ggml_nbytes(t_pos2)); } if (c_mask_seq != SEQ) { @@ -566,12 +566,12 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { const auto tp0 = std::chrono::steady_clock::now(); if (ggml_backend_graph_compute(backend, lg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): LM compute failed\n"); return {}; } - stats.ms_prefill = std::chrono::duration(std::chrono::steady_clock::now() - tp0).count(); + stats.ms_prefill = std::chrono::duration(std::chrono::steady_clock::now()-tp0).count(); if (dump_prefix) { dump_t("eagle", eagle); dump_t("conditioning", conditioning); } - ggml_backend_tensor_get(conditioning, cond_host.data(), 0, cond_host.size() * sizeof(float)); + ggml_backend_tensor_get(conditioning, cond_host.data(), 0, cond_host.size()*sizeof(float)); } // Dumping adds graph outputs, so it always rebuilds. @@ -579,7 +579,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { if (dump_prefix) head_graph.release(); const HeadKey hkey{ num_steps }; - const bool head_built = head_graph.ensure(backend, hkey, (size_t) 256 * 1024 * 1024, + const bool head_built = head_graph.ensure(backend, hkey, (size_t) 256*1024*1024, [&](ggml_context * C, HeadIO & gio) -> ggml_cgraph * { ggml_tensor * t_cond = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, num_future); ggml_set_input(t_cond); ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, state_dim, 1); ggml_set_input(t_state); @@ -594,7 +594,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * state_features = ggml_add(C, ggml_mul_mat(C, se_l2W, ggml_relu(C, ggml_add(C, ggml_mul_mat(C, se_l1W, t_state), se_l1b))), se_l2b); ggml_tensor * future = future_tokens; - const float dt = 1.0f / (float) num_steps; + const float dt = 1.0f/(float) num_steps; step_seq.assign(num_steps, nullptr); step_pred.assign(num_steps, nullptr); step_vel.assign(num_steps, nullptr); step_act.assign(num_steps, nullptr); @@ -612,7 +612,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { step_seq[s] = seq; ggml_tensor * x = seq; for (int64_t i = 0; i < dit_layers; ++i) { - ggml_tensor * enc = (i % 2 == 0) ? t_cond : nullptr; + ggml_tensor * enc = (i%2 == 0) ? t_cond : nullptr; x = dit.block(C, dit.blk[i], x, temb, enc); } @@ -623,7 +623,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * model_output = ggml_add(C, ggml_mul_mat(C, dit.po2W, h_mod), dit.po2b); step_pred[s] = model_output; - ggml_tensor * last = ggml_cont(C, ggml_view_2d(C, model_output, OUTD, AH, model_output->nb[1], (size_t) (Nseq - AH) * model_output->nb[1])); + ggml_tensor * last = ggml_cont(C, ggml_view_2d(C, model_output, OUTD, AH, model_output->nb[1], (size_t) (Nseq-AH)*model_output->nb[1])); ggml_tensor * vel = ggml_add(C, ggml_mul_mat(C, ad_l2W, ggml_relu(C, ggml_add(C, ggml_mul_mat(C, ad_l1W, last), ad_l1b))), ad_l2b); step_vel[s] = vel; actions = ggml_add(C, actions, ggml_scale(C, vel, dt)); @@ -671,8 +671,8 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { const auto td0 = std::chrono::steady_clock::now(); if (ggml_backend_graph_compute(backend, hg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): head compute failed\n"); return {}; } - stats.ms_denoise = std::chrono::duration(std::chrono::steady_clock::now() - td0).count(); - stats.ms_inference = stats.ms_prefill + stats.ms_denoise; + stats.ms_denoise = std::chrono::duration(std::chrono::steady_clock::now()-td0).count(); + stats.ms_inference = stats.ms_prefill+stats.ms_denoise; if (dump_prefix) for (int64_t s = 0; s < num_steps; ++s) { char nm[48]; @@ -683,8 +683,8 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { } std::vector out((size_t) AH * AD); - ggml_backend_tensor_get(actions, out.data(), 0, out.size() * sizeof(float)); - stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ggml_backend_tensor_get(actions, out.data(), 0, out.size()*sizeof(float)); + stats.ms_total = std::chrono::duration(std::chrono::steady_clock::now()-t0).count(); return out; } diff --git a/src/modules/preprocess.h b/src/modules/preprocess.h index b91ec7a..880c7b7 100644 --- a/src/modules/preprocess.h +++ b/src/modules/preprocess.h @@ -36,15 +36,15 @@ inline bool view_is_side(const void * data, int w, int h, int64_t side) { // src [embed, n_patches] row-major (patch p, channel e) -> dst [embed*s^2, (grid/s)^2]. inline void pixel_shuffle_hf(const float * src, float * dst, int64_t embed, int64_t grid, int64_t s) { - const int64_t g2 = grid / s, c4 = embed * s * s; + const int64_t g2 = grid/s, c4 = embed * s * s; for (int64_t h2 = 0; h2 < g2; ++h2) for (int64_t w2 = 0; w2 < g2; ++w2) { - const int64_t t = h2 * g2 + w2; + const int64_t t = h2*g2+w2; for (int64_t hs = 0; hs < s; ++hs) for (int64_t ws = 0; ws < s; ++ws) { - const int64_t p = (h2 * s + hs) * grid + (w2 * s + ws); - const int64_t base = (hs * s + ws) * embed; - std::memcpy(dst + t * c4 + base, src + p * embed, + const int64_t p = (h2*s+hs)*grid+(w2*s+ws); + const int64_t base = (hs * s+ws)*embed; + std::memcpy(dst+t * c4+base, src+p * embed, (size_t) embed * sizeof(float)); } } @@ -60,16 +60,16 @@ inline bool preprocess_image_chw(const char * arch, const ImageView & v, int64_t arch, v.w, v.h, (long long) side, (long long) side); return false; } - out.assign((size_t) 3 * side * side, 0.0f); + out.assign((size_t) 3*side * side, 0.0f); for (int64_t h = 0; h < side; ++h) for (int64_t w = 0; w < side; ++w) for (int64_t c = 0; c < 3; ++c) { float px; if (v.format == PixelFormat::U8) - px = ((const uint8_t *) v.data)[(h * side + w) * 3 + c] / 255.0f; + px = ((const uint8_t *) v.data)[(h * side+w)*3+c]/255.0f; else - px = ((const float *) v.data)[(h * side + w) * 3 + c]; - out[c * side * side + h * side + w] = px * 2.0f - 1.0f; + px = ((const float *) v.data)[(h * side+w)*3+c]; + out[c * side * side+h * side+w] = px*2.0f-1.0f; } return true; } diff --git a/src/modules/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h index a6233ac..12db2e6 100644 --- a/src/modules/qwen3vl_vit.h +++ b/src/modules/qwen3vl_vit.h @@ -91,12 +91,12 @@ struct Qwen3VLTower { inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml_tensor * x, ggml_tensor * cos_t, ggml_tensor * sin_t, int64_t seq, int64_t heads, int64_t hd, int64_t hidden, float ln_eps) { - const float scale = 1.0f / std::sqrt((float) hd); + const float scale = 1.0f/std::sqrt((float) hd); ggml_tensor * n1 = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.ln1w), w.ln1b); ggml_tensor * qkv = ggml_add(C, ggml_mul_mat(C, w.Wqkv, n1), w.bqkv); ggml_tensor * q = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], 0)); ggml_tensor * k = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], (size_t) hidden * qkv->nb[0])); - ggml_tensor * v = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], (size_t) 2 * hidden * qkv->nb[0])); + ggml_tensor * v = ggml_cont(C, ggml_view_2d(C, qkv, hidden, seq, qkv->nb[1], (size_t) 2*hidden * qkv->nb[0])); ggml_tensor * Q = rope_2d(C, to_heads(C, q, hd, heads, seq), cos_t, sin_t); ggml_tensor * K = rope_2d(C, to_heads(C, k, hd, heads, seq), cos_t, sin_t); ggml_tensor * att; @@ -113,7 +113,7 @@ inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml // pre_merge normalizes before the reshape, the deepstack taps after. inline ggml_tensor * build_merger(ggml_context * C, const MergerW & w, ggml_tensor * x, int64_t hidden, int64_t merge2, float ln_eps, bool pre_merge) { - const int64_t n_patches = x->ne[1], c_merged = hidden * merge2 * merge2, n_merged = n_patches / (merge2 * merge2); + const int64_t n_patches = x->ne[1], c_merged = hidden * merge2*merge2, n_merged = n_patches/(merge2*merge2); ggml_tensor * m; if (pre_merge) { ggml_tensor * xn = ggml_add(C, ggml_mul(C, ggml_norm(C, x, ln_eps), w.nw), w.nb); @@ -130,30 +130,30 @@ inline ggml_tensor * build_merger(ggml_context * C, const MergerW & w, ggml_tens inline void merge_block_coords(int64_t gh, int64_t gw, int64_t m, std::vector & row, std::vector & col) { const int64_t S = gh * gw; row.assign(S, 0); col.assign(S, 0); for (int64_t s = 0; s < S; ++s) { - int64_t t = s; const int64_t wj = t % m; t /= m; const int64_t wi = t % m; t /= m; - const int64_t bc = t % (gw / m); t /= (gw / m); const int64_t br = t; - row[s] = br * m + wi; col[s] = bc * m + wj; + int64_t t = s; const int64_t wj = t%m; t /= m; const int64_t wi = t%m; t /= m; + const int64_t bc = t%(gw/m); t /= (gw/m); const int64_t br = t; + row[s] = br * m+wi; col[s] = bc * m+wj; } } inline void vit_rope_tables(const std::vector & row, const std::vector & col, int64_t hd, double theta, std::vector & cos_t, std::vector & sin_t) { - const int64_t S = (int64_t) row.size(), nf = hd / 4; + const int64_t S = (int64_t) row.size(), nf = hd/4; std::vector invf(nf); for (int64_t i = 0; i < nf; ++i) - invf[i] = 1.0 / std::pow(theta, (double)(2 * i) / (double)(hd / 2)); + invf[i] = 1.0/std::pow(theta, (double)(2*i)/(double)(hd/2)); cos_t.assign((size_t) S * hd, 0.0f); sin_t.assign((size_t) S * hd, 0.0f); for (int64_t s = 0; s < S; ++s) { std::vector emb(hd); for (int64_t i = 0; i < nf; ++i) { - emb[i] = (double) row[s] * invf[i]; - emb[nf + i] = (double) col[s] * invf[i]; + emb[i] = (double) row[s]*invf[i]; + emb[nf+i] = (double) col[s]*invf[i]; } - for (int64_t i = 0; i < hd / 2; ++i) - emb[hd / 2 + i] = emb[i]; + for (int64_t i = 0; i < hd/2; ++i) + emb[hd/2+i] = emb[i]; for (int64_t i = 0; i < hd; ++i) { - cos_t[s * hd + i] = (float) std::cos(emb[i]); - sin_t[s * hd + i] = (float) std::sin(emb[i]); + cos_t[s * hd+i] = (float) std::cos(emb[i]); + sin_t[s * hd+i] = (float) std::sin(emb[i]); } } } @@ -164,20 +164,20 @@ inline void interp_pos_embed(const std::vector & table, int64_t num_side, std::vector & out) { const int64_t S = (int64_t) row.size(); out.assign((size_t) S * hidden, 0.0f); - auto src_coord = [&](int64_t k, int64_t g) -> double { return (g <= 1) ? 0.0 : (double) k * (double)(num_side - 1) / (double)(g - 1); }; + auto src_coord = [&](int64_t k, int64_t g) -> double { return (g <= 1) ? 0.0 : (double) k * (double)(num_side-1)/(double)(g-1); }; for (int64_t s = 0; s < S; ++s) { // Clamped, not just h1/w1: a grid that the spatial merge does not divide // pushes row/col past gh-1 and would index off the end of the table. - const double lim = (double) (num_side - 1); + const double lim = (double) (num_side-1); const double hy = std::min(src_coord(row[s], gh), lim), wx = std::min(src_coord(col[s], gw), lim); const int64_t h0 = (int64_t) std::floor(hy), w0 = (int64_t) std::floor(wx); - const int64_t h1 = std::min(h0 + 1, num_side - 1), w1 = std::min(w0 + 1, num_side - 1); - const double dh = hy - h0, dw = wx - w0; - const double c00 = (1 - dh) * (1 - dw), c01 = (1 - dh) * dw, c10 = dh * (1 - dw), c11 = dh * dw; - const float * T00 = &table[(h0 * num_side + w0) * hidden]; const float * T01 = &table[(h0 * num_side + w1) * hidden]; - const float * T10 = &table[(h1 * num_side + w0) * hidden]; const float * T11 = &table[(h1 * num_side + w1) * hidden]; + const int64_t h1 = std::min(h0+1, num_side-1), w1 = std::min(w0+1, num_side-1); + const double dh = hy-h0, dw = wx-w0; + const double c00 = (1-dh)*(1-dw), c01 = (1-dh)*dw, c10 = dh * (1-dw), c11 = dh * dw; + const float * T00 = &table[(h0*num_side+w0)*hidden]; const float * T01 = &table[(h0*num_side+w1)*hidden]; + const float * T10 = &table[(h1*num_side+w0)*hidden]; const float * T11 = &table[(h1*num_side+w1)*hidden]; for (int64_t c = 0; c < hidden; ++c) - out[s * hidden + c] = (float)(c00 * T00[c] + c01 * T01[c] + c10 * T10[c] + c11 * T11[c]); + out[s * hidden+c] = (float)(c00*T00[c]+c01*T01[c]+c10*T10[c]+c11*T11[c]); } } @@ -190,20 +190,20 @@ inline bool preprocess_image_patches(const char * arch, const ImageView & v, int arch, v.w, v.h, (long long) side, (long long) side); return false; } - const int64_t S = (int64_t) row.size(), pf = 3 * tps * ps * ps; + const int64_t S = (int64_t) row.size(), pf = 3*tps * ps * ps; out.assign((size_t) pf * S, 0.0f); auto px = [&](int64_t r, int64_t c, int64_t ch) -> float { if (v.format == PixelFormat::U8) - return ((const uint8_t *) v.data)[(r * side + c) * 3 + ch] / 255.0f; - return ((const float *) v.data)[(r * side + c) * 3 + ch]; + return ((const uint8_t *) v.data)[(r * side+c)*3+ch]/255.0f; + return ((const float *) v.data)[(r * side+c)*3+ch]; }; for (int64_t s = 0; s < S; ++s) for (int64_t ch = 0; ch < 3; ++ch) for (int64_t ph = 0; ph < ps; ++ph) for (int64_t pw = 0; pw < ps; ++pw) { - const float val = (px(row[s] * ps + ph, col[s] * ps + pw, ch) - QWEN3VL_MEAN[ch]) / QWEN3VL_STD[ch]; + const float val = (px(row[s]*ps+ph, col[s]*ps+pw, ch)-QWEN3VL_MEAN[ch])/QWEN3VL_STD[ch]; for (int64_t t = 0; t < tps; ++t) - out[s * pf + ch * tps * ps * ps + t * ps * ps + ph * ps + pw] = val; + out[s * pf+ch * tps * ps * ps+t * ps * ps+ph * ps+pw] = val; } return true; } diff --git a/src/serving/hf_fetch.h b/src/serving/hf_fetch.h index 69c664b..3ba3f57 100644 --- a/src/serving/hf_fetch.h +++ b/src/serving/hf_fetch.h @@ -84,7 +84,7 @@ inline std::string hf_resolve(const std::string & spec) { const size_t colon = spec.find(':'); const std::string repo = spec.substr(0, colon); - const std::string file = (colon == std::string::npos) ? "" : spec.substr(colon + 1); + const std::string file = (colon == std::string::npos) ? "" : spec.substr(colon+1); if (repo.find('/') == std::string::npos || !hf_token_ok(repo, true) || (!file.empty() && !hf_token_ok(file, false))) { @@ -92,7 +92,7 @@ inline std::string hf_resolve(const std::string & spec) { return ""; } - const fs::path dir = fs::path(hf_cache_root()) / repo; + const fs::path dir = fs::path(hf_cache_root())/repo; std::error_code ec; if (fs::is_directory(dir, ec)) { diff --git a/src/serving/server.cpp b/src/serving/server.cpp index 2ba432c..733140f 100644 --- a/src/serving/server.cpp +++ b/src/serving/server.cpp @@ -90,7 +90,7 @@ bool decode_image(const vla::Image & img, stbi_image_free(px); return false; } - u8.assign(px, px + size_t(3) * w * h); + u8.assign(px, px+size_t(3)*w * h); stbi_image_free(px); view = { u8.data(), w, h, vla::PixelFormat::U8 }; return true; @@ -101,14 +101,14 @@ bool decode_image(const vla::Image & img, img.width(), img.height(), kMaxImageDim); return false; } - const size_t expected = size_t(3) * img.width() * img.height(); + const size_t expected = size_t(3)*img.width()*img.height(); if (img.data().size() != expected) { std::fprintf(stderr, "vla-server: RGB_U8 size %zu != 3*%u*%u = %zu\n", img.data().size(), img.width(), img.height(), expected); return false; } u8.assign(reinterpret_cast(img.data().data()), - reinterpret_cast(img.data().data()) + expected); + reinterpret_cast(img.data().data())+expected); view = { u8.data(), int(img.width()), int(img.height()), vla::PixelFormat::U8 }; return true; } else if (img.encoding() == vla::Image::F32_RGB_01) { @@ -118,7 +118,7 @@ bool decode_image(const vla::Image & img, img.width(), img.height(), kMaxImageDim); return false; } - const size_t pixels = size_t(3) * img.width() * img.height(); + const size_t pixels = size_t(3)*img.width()*img.height(); const size_t expected = pixels * sizeof(float); if (img.data().size() != expected) { std::fprintf(stderr, "vla-server: F32_RGB_01 size %zu != 4*3*%u*%u = %zu\n", @@ -221,13 +221,13 @@ int main(int argc, char ** argv) { std::vector positionals; for (int i = 1; i < argc; ++i) { std::string a = argv[i]; - if (a == "--bind" && i + 1 < argc) { + if (a == "--bind" && i+1 < argc) { bind_addr = argv[++i]; - } else if (a == "-hf" && i + 1 < argc) { + } else if (a == "-hf" && i+1 < argc) { hf_spec = argv[++i]; - } else if (a == "--config" && i + 1 < argc) { + } else if (a == "--config" && i+1 < argc) { config_path = argv[++i]; - } else if (a == "--timing-detail" && i + 1 < argc) { + } else if (a == "--timing-detail" && i+1 < argc) { const std::string v = argv[++i]; if (v == "none") timing_detail = vla::TimingDetail::NONE; @@ -299,7 +299,7 @@ int main(int argc, char ** argv) { sock.set(zmq::sockopt::linger, 0); // 64 MiB is above any real request (16 views of 512x512 F32 RGB is ~50 MiB) and // low enough to bound protobuf's expansion during ParseFromArray. - sock.set(zmq::sockopt::maxmsgsize, int64_t(64) * 1024 * 1024); + sock.set(zmq::sockopt::maxmsgsize, int64_t(64)*1024*1024); // A peer that sends a frame with SNDMORE and then stalls would otherwise park // this single-threaded loop in recv for good, starving every other client. sock.set(zmq::sockopt::rcvtimeo, 5000); @@ -424,7 +424,7 @@ int main(int argc, char ** argv) { send_reply(make_error_response(rid, buf)); continue; } - const int expected_noise_n = int(cfg.n_suffix * cfg.max_action_dim); + const int expected_noise_n = int(cfg.n_suffix*cfg.max_action_dim); if (req.noise_size() != 0 && req.noise_size() != expected_noise_n) { char buf[128]; std::snprintf(buf, sizeof(buf), "noise length %d != 0 or %d (chunk_size * action_dim)", @@ -444,7 +444,7 @@ int main(int argc, char ** argv) { if (use_precomputed) { precomputed_n_views = static_cast(req.precomputed_img_emb_n_views()); - const int64_t per_view = cfg.n_img * cfg.hidden; + const int64_t per_view = cfg.n_img*cfg.hidden; const int64_t expected = per_view * static_cast(precomputed_n_views); if (precomputed_n_views < 1 || static_cast(req.precomputed_img_emb_size()) != expected) { @@ -554,9 +554,9 @@ int main(int argc, char ** argv) { send_reply(body); ++served; - if (served % 10 == 1) { + if (served%10 == 1) { const float ms_other = std::max(0.f, - st.ms_total - st.ms_vision - st.ms_inference); + st.ms_total-st.ms_vision-st.ms_inference); if (timing_detail == vla::TimingDetail::PHASE) { std::printf("vla-server: rid=%llu served=%llu total=%.1f ms " "vision=%.1f inf=%.1f (prefill=%.1f + denoise=%.1f) other=%.1f\n", diff --git a/src/serving/vla-bench.cpp b/src/serving/vla-bench.cpp index d51b208..46da986 100644 --- a/src/serving/vla-bench.cpp +++ b/src/serving/vla-bench.cpp @@ -49,9 +49,9 @@ void usage(const char * prog) { double percentile(const std::vector & v, double p) { if (v.empty()) return 0.0; - const double idx = p * (double) (v.size() - 1); + const double idx = p * (double) (v.size()-1); const size_t lo = (size_t) std::floor(idx), hi = (size_t) std::ceil(idx); - return v[lo] + (v[hi] - v[lo]) * (idx - (double) lo); + return v[lo]+(v[hi]-v[lo])*(idx-(double) lo); } } // namespace @@ -65,7 +65,7 @@ int main(int argc, char ** argv) { for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto need = [&](const char * name) -> const char * { - if (i + 1 >= argc) { + if (i+1 >= argc) { std::fprintf(stderr, "vla-bench: %s needs a value\n", name); std::exit(1); } @@ -114,7 +114,7 @@ int main(int argc, char ** argv) { } if (label.empty()) { const size_t slash = ckpt.find_last_of('/'); - label = (slash == std::string::npos) ? ckpt : ckpt.substr(slash + 1); + label = (slash == std::string::npos) ? ckpt : ckpt.substr(slash+1); } vla::Model * m = vla::model_load(mmproj, ckpt, ""); @@ -124,29 +124,29 @@ int main(int argc, char ** argv) { } const vla::Config & cfg = vla::model_config(m); - std::vector> pixels(n_images, std::vector((size_t) 3 * side * side)); + std::vector> pixels(n_images, std::vector((size_t) 3*side * side)); std::vector views(n_images); for (int v = 0; v < n_images; ++v) { for (int y = 0; y < side; ++y) for (int x = 0; x < side; ++x) for (int c = 0; c < 3; ++c) - pixels[v][((size_t) y * side + x) * 3 + c] = (uint8_t) ((x + 2 * y + 40 * c + 17 * v) & 0xFF); + pixels[v][((size_t) y * side+x)*3+c] = (uint8_t) ((x+2*y+40*c+17*v) & 0xFF); views[v] = vla::ImageView{ pixels[v].data(), side, side, vla::PixelFormat::U8 }; } std::vector lang((size_t) n_tokens); for (int i = 0; i < n_tokens; ++i) - lang[i] = 1 + (i % 100); + lang[i] = 1+(i%100); if (extra_token >= 0 && extra_count > 0) lang.insert(lang.end(), (size_t) extra_count, extra_token); std::vector state((size_t) cfg.max_state_dim, 0.0f); for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) - state[i] = 0.01f * (float) (i + 1); + state[i] = 0.01f * (float) (i+1); - std::vector noise((size_t) cfg.max_action_dim * (size_t) cfg.n_suffix); + std::vector noise((size_t) cfg.max_action_dim*(size_t) cfg.n_suffix); for (size_t i = 0; i < noise.size(); ++i) - noise[i] = 0.001f * (float) ((i * 2654435761u) % 1000) - 0.5f; + noise[i] = 0.001f * (float) ((i*2654435761u)%1000)-0.5f; vla::Inputs in{}; in.images = views.data(); @@ -176,7 +176,7 @@ int main(int argc, char ** argv) { vla::model_free(m); return 1; } - ms.push_back(std::chrono::duration(t1 - t0).count()); + ms.push_back(std::chrono::duration(t1-t0).count()); vision_sum += vla::last_stats(m).ms_vision; } @@ -184,7 +184,7 @@ int main(int argc, char ** argv) { const double lo = ms.front(); const double p50 = percentile(ms, 0.50); const double p90 = percentile(ms, 0.90); - const double vision = vision_sum / (double) reps; + const double vision = vision_sum/(double) reps; if (markdown) { std::printf("| %s | %d | %d | %d | %.1f | %.1f | %.1f | %.1f |\n", diff --git a/src/serving/vla-cli.cpp b/src/serving/vla-cli.cpp index 9030e3c..81156d2 100644 --- a/src/serving/vla-cli.cpp +++ b/src/serving/vla-cli.cpp @@ -57,9 +57,9 @@ bool parse_ints(const std::string & s, std::vector & out) { break; errno = 0; char * e = nullptr; - long long x = std::strtoll(s.c_str() + i, &e, 10); - if (e == s.c_str() + i) { - std::fprintf(stderr, "vla-cli: bad token near '%s'\n", s.c_str() + i); + long long x = std::strtoll(s.c_str()+i, &e, 10); + if (e == s.c_str()+i) { + std::fprintf(stderr, "vla-cli: bad token near '%s'\n", s.c_str()+i); return false; } if (errno == ERANGE || x < INT32_MIN || x > INT32_MAX) { @@ -67,7 +67,7 @@ bool parse_ints(const std::string & s, std::vector & out) { return false; } out.push_back((int32_t) x); - i = (size_t) (e - s.c_str()); + i = (size_t) (e-s.c_str()); } return true; } @@ -82,9 +82,9 @@ bool parse_floats(const std::string & s, std::vector & out) { if (i >= s.size()) break; char * e = nullptr; - float x = std::strtof(s.c_str() + i, &e); - if (e == s.c_str() + i) { - std::fprintf(stderr, "vla-cli: bad number near '%s'\n", s.c_str() + i); + float x = std::strtof(s.c_str()+i, &e); + if (e == s.c_str()+i) { + std::fprintf(stderr, "vla-cli: bad number near '%s'\n", s.c_str()+i); return false; } if (!std::isfinite(x)) { @@ -92,7 +92,7 @@ bool parse_floats(const std::string & s, std::vector & out) { return false; } out.push_back(x); - i = (size_t) (e - s.c_str()); + i = (size_t) (e-s.c_str()); } return true; } @@ -105,7 +105,7 @@ bool load_image(const char * path, std::vector & buf, int & w, int & h) std::fprintf(stderr, "vla-cli: cannot load image %s: %s\n", path, stbi_failure_reason()); return false; } - buf.assign(px, px + size_t(3) * w * h); + buf.assign(px, px+size_t(3)*w * h); stbi_image_free(px); return true; } @@ -215,7 +215,7 @@ int main(int argc, char ** argv) { for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto need = [&](const char * name) -> const char * { - if (i + 1 >= argc) { + if (i+1 >= argc) { std::fprintf(stderr, "vla-cli: %s needs a value\n", name); std::exit(1); } @@ -315,7 +315,7 @@ int main(int argc, char ** argv) { const int64_t cols = cfg.max_action_dim > 0 ? cfg.max_action_dim : 1; if (pretty) { for (size_t i = 0; i < act.size(); ++i) - std::printf("%.6g%c", act[i], ((int64_t) (i + 1) % cols == 0) ? '\n' : ' '); + std::printf("%.6g%c", act[i], ((int64_t) (i+1)%cols == 0) ? '\n' : ' '); } else { std::printf("action_len=%zu\n", act.size()); for (float x : act) diff --git a/src/serving/vlm-server.cpp b/src/serving/vlm-server.cpp index 05dff9e..ea92995 100644 --- a/src/serving/vlm-server.cpp +++ b/src/serving/vlm-server.cpp @@ -77,11 +77,11 @@ int main(int argc, char ** argv) { for (int i = 1; i < argc; ++i) { std::string a = argv[i]; - if (a == "--bind" && i + 1 < argc) { + if (a == "--bind" && i+1 < argc) { bind_addr = argv[++i]; - } else if ((a == "-c" || a == "--n-ctx") && i + 1 < argc) { + } else if ((a == "-c" || a == "--n-ctx") && i+1 < argc) { lp.n_ctx = std::atoi(argv[++i]); - } else if (a == "--ngl" && i + 1 < argc) { + } else if (a == "--ngl" && i+1 < argc) { lp.n_gpu_layers = std::atoi(argv[++i]); } else if (a == "--no-mmproj-gpu") { lp.mmproj_use_gpu = false; @@ -113,7 +113,7 @@ int main(int argc, char ** argv) { zmq::socket_t sock(zctx, zmq::socket_type::router); sock.set(zmq::sockopt::linger, 0); // Per frame only; the recv loop caps the multipart total. - sock.set(zmq::sockopt::maxmsgsize, int64_t(64) * 1024 * 1024); + sock.set(zmq::sockopt::maxmsgsize, int64_t(64)*1024*1024); // A peer that sends a frame with SNDMORE and then stalls would otherwise park // this single-threaded loop in recv for good, starving every other client. sock.set(zmq::sockopt::rcvtimeo, 5000); @@ -152,7 +152,7 @@ int main(int argc, char ** argv) { // maxmsgsize bounds each frame but not how many, so a peer could stream // sub-limit frames until memory runs out. constexpr size_t kMaxEnvFrames = 8; - constexpr size_t kMaxEnvBytes = 64 * 1024; + constexpr size_t kMaxEnvBytes = 64*1024; std::vector env; std::string payload; @@ -221,7 +221,7 @@ int main(int argc, char ** argv) { // One 60 MiB payload of tiny messages would cost template formatting and // tokenization far beyond anything n_ctx could consume. constexpr int kMaxMessages = 512; - constexpr size_t kMaxTextBytes = 4u * 1024 * 1024; + constexpr size_t kMaxTextBytes = 4u*1024*1024; if (req.messages_size() > kMaxMessages) { send_reply(make_error_stream(rid, "too many messages (max 512)")); continue; @@ -273,7 +273,7 @@ int main(int argc, char ** argv) { send_reply(make_error_stream(rid, buf)); decode_ok = false; break; } - const size_t expected = size_t(3) * im.width() * im.height(); + const size_t expected = size_t(3)*im.width()*im.height(); if (im.data().size() != expected) { char buf[96]; std::snprintf(buf, sizeof(buf), "image[%d] RGB_U8 size %zu != 3*%u*%u", v, @@ -329,7 +329,7 @@ int main(int argc, char ** argv) { const auto t0 = std::chrono::steady_clock::now(); const vlm::ChatResult r = engine.chat(messages, images, sp, on_token); const float ms_total = std::chrono::duration( - std::chrono::steady_clock::now() - t0).count(); + std::chrono::steady_clock::now()-t0).count(); vlm_chat::StreamMessage sm; vlm_chat::ChatResponse * resp = sm.mutable_final(); diff --git a/src/vla_c_api.cpp b/src/vla_c_api.cpp index 3c38fdf..339c016 100644 --- a/src/vla_c_api.cpp +++ b/src/vla_c_api.cpp @@ -157,10 +157,10 @@ int32_t vla_predict(vla_model * h, const vla_inputs * in, return VLA_ERR_PREDICT; // malloc pairs with vla_free_actions, which callers may replace. - float * buf = (float *) std::malloc(act.size() * sizeof(float)); + float * buf = (float *) std::malloc(act.size()*sizeof(float)); if (!buf) return VLA_ERR_EXCEPTION; - std::memcpy(buf, act.data(), act.size() * sizeof(float)); + std::memcpy(buf, act.data(), act.size()*sizeof(float)); *out_actions = buf; *out_n = (int64_t) act.size(); return VLA_OK; diff --git a/src/vlm/engine.cpp b/src/vlm/engine.cpp index 57cfd34..42dcd64 100644 --- a/src/vlm/engine.cpp +++ b/src/vlm/engine.cpp @@ -126,7 +126,7 @@ bool bitmap_to_image(mtmd::bitmap & bmp, Image & out) { } out.width = bmp.nx(); out.height = bmp.ny(); - out.rgb.assign(bmp.data(), bmp.data() + bmp.n_bytes()); + out.rgb.assign(bmp.data(), bmp.data()+bmp.n_bytes()); return true; } } @@ -180,14 +180,14 @@ ChatResult Engine::chat(const std::vector & messages, } int img_msg_idx = -1; - for (int i = (int) messages.size() - 1; i >= 0; --i) { + for (int i = (int) messages.size()-1; i >= 0; --i) { if (messages[i].role == "user") { img_msg_idx = i; break; } } if (img_msg_idx < 0) { - img_msg_idx = (int) messages.size() - 1; + img_msg_idx = (int) messages.size()-1; } const char * marker = mtmd_default_marker(); @@ -205,10 +205,10 @@ ChatResult Engine::chat(const std::vector & messages, std::string prefix; for (size_t k = 0; k < images.size(); ++k) prefix += marker; - msg.content = prefix + msg.content; + msg.content = prefix+msg.content; } for (const auto & im : images) { - if (im.rgb.size() != (size_t) im.width * im.height * 3) { + if (im.rgb.size() != (size_t) im.width*im.height*3) { res.finish_reason = "error"; res.error = "image rgb size != w*h*3"; common_sampler_free(smpl); @@ -254,7 +254,7 @@ ChatResult Engine::chat(const std::vector & messages, n_past = new_n_past; } res.prompt_tokens = (int32_t) n_past; - res.ms_prefill = (ggml_time_us() - t_prefill_start) / 1000.0f; + res.ms_prefill = (ggml_time_us()-t_prefill_start)/1000.0f; const int n_predict = sampling.max_tokens <= 0 ? INT_MAX : sampling.max_tokens; std::vector generated; @@ -286,7 +286,7 @@ ChatResult Engine::chat(const std::vector & messages, } } res.completion_tokens = (int32_t) generated.size(); - res.ms_decode = (ggml_time_us() - t_decode_start) / 1000.0f; + res.ms_decode = (ggml_time_us()-t_decode_start)/1000.0f; if (res.finish_reason != "error") { res.text = common_detokenize(impl_->lctx, generated, false); diff --git a/src/vlm/engine.h b/src/vlm/engine.h index 7028ef4..b8e2bae 100644 --- a/src/vlm/engine.h +++ b/src/vlm/engine.h @@ -14,7 +14,7 @@ /** * @file engine.h - * @brief Chat / image-grounded text generation built on llama.cpp. + * @brief Chat/image-grounded text generation built on llama.cpp. * * The @ref vlm namespace is independent of the @c vla action-prediction * pipeline: it exposes a small "load model, send messages + images, get From c45fc17ffd8bed7633258f5e6e2ebe6000166971 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 13 Aug 2026 23:24:36 +0700 Subject: [PATCH 17/21] tighten for-header spacing across src --- src/cuda/vla_cuda_bf16.cu | 18 ++--- src/env_flag.h | 2 +- src/gguf_reader.h | 4 +- src/kernels/bitvla/bitnet_kernels.h | 46 ++++++------- src/kernels/bitvla/bitvla_fp32head_cuda.cu | 14 ++-- src/kernels/bitvla/bitvla_lm_cuda.cu | 56 +++++++-------- src/kernels/bitvla/bitvla_vit_cuda.cu | 2 +- src/loader.cpp | 2 +- src/models/bitvla.cpp | 68 +++++++++--------- src/models/dit_common.h | 10 +-- src/models/evo1.cpp | 60 ++++++++-------- src/models/gr00tn1d5.cpp | 16 ++--- src/models/gr00tn1d6.cpp | 18 ++--- src/models/gr00tn1d7.cpp | 68 +++++++++--------- src/models/openvla_oft.cpp | 6 +- src/models/pi0.cpp | 32 ++++----- src/models/pi05.cpp | 24 +++---- src/models/smolvla.cpp | 80 +++++++++++----------- src/models/vla_adapter.cpp | 6 +- src/models/vla_jepa.cpp | 56 +++++++-------- src/modules/preprocess.h | 34 ++++----- src/modules/prompt.cpp | 10 +-- src/modules/qwen3vl_vit.h | 28 ++++---- src/serving/server.cpp | 10 +-- src/serving/vla-bench.cpp | 20 +++--- src/serving/vla-cli.cpp | 6 +- src/serving/vlm-server.cpp | 4 +- src/vla_c_api.cpp | 2 +- src/vlm/engine.cpp | 10 +-- 29 files changed, 356 insertions(+), 356 deletions(-) diff --git a/src/cuda/vla_cuda_bf16.cu b/src/cuda/vla_cuda_bf16.cu index 72cf2e3..70a3d63 100644 --- a/src/cuda/vla_cuda_bf16.cu +++ b/src/cuda/vla_cuda_bf16.cu @@ -162,7 +162,7 @@ __global__ void k_bin_bcast_bf16_vec8( __nv_bfloat16*av = reinterpret_cast<__nv_bfloat16 *>(&a); #pragma unroll - for (int k = 0; k < 8; ++k) { + for (int k=0; k<8; ++k) { const float b = (float) r1[i0+k]; av[k] = f2bf(apply_bin(op, bf2f(av[k]), b)); } @@ -381,7 +381,7 @@ __global__ void k_fused_bin_bcast_bf16( const int64_t j = row1+bcast_idx(i0, ne10, ne0)*s10; float acc = bf2f(r0[i0*s00]); - for (int k = 0; k < n_fuse; ++k) { + for (int k=0; ksrc[k+1]; if (!s || s->type != src1->type) return false; if (!ggml_are_same_shape(s, src1)) return false; - for (int d = 0; d < GGML_MAX_DIMS; ++d) { + for (int d=0; dnb[d] != src1->nb[d]) return false; } @@ -434,7 +434,7 @@ bool fused_bin_bcast(ggml_tensor * dst, int n_fuse, cudaStream_t stream) { #define VLA_LAUNCH_FUSED(TYPE) \ do { \ SrcPtrs srcs{}; \ - for (int k = 0; k < n_fuse; ++k) srcs.p[k] = (const TYPE *) dst->src[k+1]->data; \ + for (int k=0; ksrc[k+1]->data; \ k_fused_bin_bcast_bf16<<>>( \ (const __nv_bfloat16 *) src0->data, srcs, n_fuse, \ (__nv_bfloat16 *) dst->data, \ @@ -538,7 +538,7 @@ __device__ inline float block_sum(float v, float * shared) { const int tid = threadIdx.x; shared[tid] = v; __syncthreads(); - for (int s = blockDim.x/2; s > 0; s >>= 1) { + for (int s=blockDim.x/2; s>0; s >>= 1) { if (tid < s) shared[tid] += shared[tid+s]; __syncthreads(); @@ -556,7 +556,7 @@ __global__ void k_norm_bf16(const __nv_bfloat16*__restrict__ x, __nv_bfloat16*__ __nv_bfloat16 * dr = dst+row*sd1; float sum = 0.0f, sumsq = 0.0f; - for (int64_t c = threadIdx.x; c < ncols; c += blockDim.x) { + for (int64_t c=threadIdx.x; c f = read_f32(name); if (f.empty()) return {}; const int64_t n = (int64_t) f.size(); - if (gemma_norm) for (int64_t i = 0; i < n; ++i) f[i] += 1.0f; + if (gemma_norm) for (int64_t i=0; i o(n * sizeof(float)); std::memcpy(o.data(), f.data(), o.size()); @@ -196,7 +196,7 @@ struct gguf_reader { const size_t elsz = (t->type == GGML_TYPE_F32) ? 4u : 2u; const size_t rb = (size_t) cols * elsz; std::vector row(rb); - for (size_t k = 0; k < row_ids.size(); ++k) { + for (size_t k=0; k= rows) { std::fprintf(stderr, "vla(%s): row %d out of range for %s\n", arch, r, name); diff --git a/src/kernels/bitvla/bitnet_kernels.h b/src/kernels/bitvla/bitnet_kernels.h index 7d758eb..bae049d 100644 --- a/src/kernels/bitvla/bitnet_kernels.h +++ b/src/kernels/bitvla/bitnet_kernels.h @@ -80,7 +80,7 @@ __device__ void decode_i2s_to_i8s(T1 *_i2s, T2 *_i8s, const int N = 16) static constexpr uint I4s_TO_I8s_MAGIC_NUM = 0x00000000; #pragma unroll - for (int i = 0; i < (N/4); i++) + for (int i=0; i<(N/4); i++) { asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(i8s[i]) @@ -117,7 +117,7 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel(int8_t* __restric int red_buf0[1]; in_thread_C_local[0] = 0; #pragma unroll - for (int k_0 = 0; k_0 < K/(K_per_loop * K_block_size); ++k_0) { + for (int k_0=0; k_0 0; offset /= 2) { + for (int offset=K_block_size/2; offset>0; offset /= 2) { red_buf0[0] += __shfl_down_sync(__activemask(), red_buf0[0], offset, K_block_size); } int out_idx = ((((int)blockIdx.x)*N_block_size)+((int)threadIdx.y)); @@ -191,12 +191,12 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( wmma::fragment b_frag; wmma::fragment acc[TILES_PER_WARP]; #pragma unroll - for (int t = 0; t < TILES_PER_WARP; ++t) + for (int t=0; t> 3; const int kk = (idx & 7)*16; @@ -216,10 +216,10 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m( __syncthreads(); #pragma unroll - for (int k16 = 0; k16 < K_CHUNK/16; ++k16) { + for (int k16=0; k16> 4; const int col = lin & 15; @@ -334,12 +334,12 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( wmma::fragment b_frag; wmma::fragment acc[M_PER_WARP]; #pragma unroll - for (int t = 0; t < M_PER_WARP; ++t) + for (int t=0; t> 3; const int kk = (idx & 7)*16; @@ -351,7 +351,7 @@ __global__ void __launch_bounds__(128) ladder_int8xint2_kernel_m_wide( // All 128 threads cooperate on one weight tile at a time, reproducing the // pack's swizzle exactly; only the tile base changes per j. #pragma unroll - for (int j = 0; j < N_TILES; ++j) { + for (int j=0; j> 4; const int col = lin & 15; @@ -475,13 +475,13 @@ __global__ void act_quant_kernel( int8_t* row_out = out+m * K; float local_max = 0.0f; - for (int k = tid; k < K; k += BLOCK_THREADS) { + for (int k=tid; k local_max) local_max = v; } - for (int off = 16; off > 0; off >>= 1) { + for (int off=16; off>0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, local_max, off); if (other > local_max) local_max = other; @@ -495,7 +495,7 @@ __global__ void act_quant_kernel( __syncthreads(); if (warp_id == 0) { float v = (tid < (BLOCK_THREADS+31)/32) ? smem[lane] : 0.0f; - for (int off = 16; off > 0; off >>= 1) { + for (int off=16; off>0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); if (other > v) v = other; @@ -509,7 +509,7 @@ __global__ void act_quant_kernel( if (tid == 0) scales[m] = scale; - for (int k = tid; k < K; k += BLOCK_THREADS) { + for (int k=tid; k 127.0f) diff --git a/src/kernels/bitvla/bitvla_fp32head_cuda.cu b/src/kernels/bitvla/bitvla_fp32head_cuda.cu index d40c20d..3bd1c24 100644 --- a/src/kernels/bitvla/bitvla_fp32head_cuda.cu +++ b/src/kernels/bitvla/bitvla_fp32head_cuda.cu @@ -100,9 +100,9 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, float* o = out+(size_t)m * K; float sum = 0.0f; - for (int k = tid; k < K; k += BLOCK) + for (int k=tid; k 0; off >>= 1) + for (int off=16; off>0; off >>= 1) sum += __shfl_down_sync(0xffffffff, sum, off); __shared__ float smem[32]; if ((tid & 31) == 0) @@ -110,7 +110,7 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -119,18 +119,18 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, const float mean = smem[0]/(float)K; float vsum = 0.0f; - for (int k = tid; k < K; k += BLOCK) { + for (int k=tid; k 0; off >>= 1) + for (int off=16; off>0; off >>= 1) vsum += __shfl_down_sync(0xffffffff, vsum, off); if ((tid & 31) == 0) smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -138,7 +138,7 @@ __global__ void layernorm_fp32_kernel(const float* __restrict__ x, __syncthreads(); const float inv_std = rsqrtf(smem[0]/(float)K+eps); - for (int k = tid; k < K; k += BLOCK) { + for (int k=tid; k 0; off >>= 1) + for (int off=16; off>0; off >>= 1) ss += __shfl_down_sync(0xffffffff, ss, off); __shared__ float smem[32]; if ((tid & 31) == 0) @@ -58,7 +58,7 @@ __global__ void rmsnorm_bf16_kernel(const __nv_bfloat16* __restrict__ x, __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -67,7 +67,7 @@ __global__ void rmsnorm_bf16_kernel(const __nv_bfloat16* __restrict__ x, const float mean = smem[0]/(float)K; const float scale = rsqrtf(mean+eps); - for (int k = tid; k < K; k += BLOCK) { + for (int k=tid; k mx) mx = v; } - for (int off = 16; off > 0; off >>= 1) { + for (int off=16; off>0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, mx, off); if (other > mx) mx = other; @@ -122,7 +122,7 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : -INFINITY; - for (int off = 16; off > 0; off >>= 1) { + for (int off=16; off>0; off >>= 1) { float other = __shfl_down_sync(0xffffffff, v, off); if (other > v) v = other; @@ -134,17 +134,17 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, const float max_v = smem[0]; float s_sum = 0.0f; - for (int i = tid; i < S; i += BLOCK) { + for (int i=tid; i 0; off >>= 1) + for (int off=16; off>0; off >>= 1) s_sum += __shfl_down_sync(0xffffffff, s_sum, off); if ((tid & 31) == 0) smem[tid >> 5] = s_sum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -152,7 +152,7 @@ __global__ void softmax_scaled_bf16_kernel(__nv_bfloat16* __restrict__ inout, __syncthreads(); const float inv_sum = 1.0f/smem[0]; - for (int i = tid; i < S; i += BLOCK) { + for (int i=tid; i h_cos((size_t)max_seq * half), h_sin((size_t)max_seq * half); - for (int s = 0; s < max_seq; ++s) { - for (int k = 0; k < half; ++k) { + for (int s=0; s tmp(n); std::vector f32(n); cudaMemcpy(tmp.data(), d_ptr, n * sizeof(__nv_bfloat16), cudaMemcpyDeviceToHost); - for (size_t i = 0; i < n; ++i) { + for (size_t i=0; i(&tmp[i]); uint32_t b = ((uint32_t) u) << 16; float f; std::memcpy(&f, &b, 4); @@ -561,9 +561,9 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, __nv_bfloat16* o = out+(size_t)m * K; float sum = 0.0f; - for (int k = tid; k < K; k += BLOCK) + for (int k=tid; k 0; off >>= 1) + for (int off=16; off>0; off >>= 1) sum += __shfl_down_sync(0xffffffff, sum, off); __shared__ float smem[32]; if ((tid & 31) == 0) @@ -571,7 +571,7 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -580,18 +580,18 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, const float mean = smem[0]/(float)K; float vsum = 0.0f; - for (int k = tid; k < K; k += BLOCK) { + for (int k=tid; k 0; off >>= 1) + for (int off=16; off>0; off >>= 1) vsum += __shfl_down_sync(0xffffffff, vsum, off); if ((tid & 31) == 0) smem[tid >> 5] = vsum; __syncthreads(); if ((tid >> 5) == 0) { float v = (tid < (BLOCK+31)/32) ? smem[tid] : 0.0f; - for (int off = 16; off > 0; off >>= 1) + for (int off=16; off>0; off >>= 1) v += __shfl_down_sync(0xffffffff, v, off); if (tid == 0) smem[0] = v; @@ -599,7 +599,7 @@ __global__ void layernorm_bias_bf16_kernel(const __nv_bfloat16* __restrict__ x, __syncthreads(); const float inv_std = rsqrtf(smem[0]/(float)K+eps); - for (int k = tid; k < K; k += BLOCK) { + for (int k=tid; khidden; h_dump.resize(n); h_dump_f32.resize(n); cudaMemcpy(h_dump.data(), d_ptr, n * sizeof(__nv_bfloat16), cudaMemcpyDeviceToHost); - for (size_t i = 0; i < n; ++i) { + for (size_t i=0; i(&h_dump[i]); uint32_t b = ((uint32_t) u) << 16; float f; std::memcpy(&f, &b, 4); @@ -704,7 +704,7 @@ extern "C" int bitvla_lm_cuda_forward(bitvla_lm_cuda_ctx* ctx, cudaStreamSynchronize(stream); dump_to_file("lm_layer_input", ctx->d_h); } - for (int L = 0; L < ctx->n_layers; ++L) { + for (int L=0; Ln_layers; ++L) { int rc = run_layer(ctx, L, seq, stream); if (rc != 0) return rc; diff --git a/src/kernels/bitvla/bitvla_vit_cuda.cu b/src/kernels/bitvla/bitvla_vit_cuda.cu index 7728639..2f48e3e 100644 --- a/src/kernels/bitvla/bitvla_vit_cuda.cu +++ b/src/kernels/bitvla/bitvla_vit_cuda.cu @@ -282,7 +282,7 @@ int bitvla_vit_cuda_forward(bitvla_vit_cuda_ctx* ctx, bitvla_add_bf16(ctx->d_h, ctx->pos_emb, ctx->d_h, seq * H, stream); - for (int L = 0; L < ctx->n_layers; ++L) { + for (int L=0; Ln_layers; ++L) { int rc = run_vit_layer(ctx, L, stream); if (rc != 0) return rc; diff --git a/src/loader.cpp b/src/loader.cpp index 1c1f6ce..9a67f05 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -141,7 +141,7 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu } *out_buf = buf; - for (ggml_tensor * t = ggml_get_first_tensor(ctx_); t; t = ggml_get_next_tensor(ctx_, t)) { + for (ggml_tensor * t=ggml_get_first_tensor(ctx_); t; t=ggml_get_next_tensor(ctx_, t)) { const char * name = ggml_get_name(t); const bool fused = std::any_of(fused_.begin(), fused_.end(), [&](const Fused & f) { return f.dst == t; }); diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index 5107c90..76e7e66 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -74,17 +74,17 @@ void bitvla_act_quant_op(ggml_tensor * dst, const ggml_tensor * a, int ith, int const int64_t r1 = std::min(rows, r0+per); const float * src = (const float *) a->data; float * out = (float *) dst->data; - for (int64_t r = r0; r < r1; ++r) { + for (int64_t r=r0; r 127.0f) q = 127.0f; @@ -432,7 +432,7 @@ static void recover_ternary_and_scale(const float* W, int64_t n, // Per-tensor absmean scale (1/mean|W|), matching scripts/bitvla_int2_pack.py; // the int2-packed path bakes the same scale. double s = 0.0; - for (int64_t i = 0; i < n; ++i) + for (int64_t i=0; i 0 ? (float) (s/(double) n) : 0.0f; if (mean < 1e-5f) @@ -440,7 +440,7 @@ static void recover_ternary_and_scale(const float* W, int64_t n, absmean = mean; const float inv = 1.0f/mean; ternary.resize(n); - for (int64_t i = 0; i < n; ++i) { + for (int64_t i=0; i 1.0f) q = 1.0f; @@ -455,7 +455,7 @@ static std::vector pack_ladder_int2(const int8_t* W, int64_t N, int64_t constexpr int WMMA_K = 32, K_PER_ITER = K_PER_LOOP * K_BLOCK; const int64_t n_slots = N * K/16; std::vector out(N * K/4, 0); - for (int64_t s = 0; s < n_slots; ++s) { + for (int64_t s=0; s pack_ladder_int2(const int8_t* W, int64_t N, int64_t const int64_t y_in_h = in_yhalf%8; const int64_t n_global = n_block * N_BLOCK+y_half*8+y_in_h; const int64_t k_sub = k_0*K_PER_ITER+major_k * WMMA_K+sub_k * K_PER_LOOP; - for (int byte_i = 0; byte_i < 4; ++byte_i) { + for (int byte_i=0; byte_i<4; ++byte_i) { uint8_t b = 0; - for (int j = 0; j < 4; ++j) { + for (int j=0; j<4; ++j) { const int t = (int) W[n_global * K+(k_sub+byte_i+4*j)]; const uint8_t enc = (uint8_t)(t+2) & 0x3; b |= (enc << (2*j)); @@ -489,7 +489,7 @@ static inline uint16_t f32_to_bf16_u16(float f) { static __nv_bfloat16* upload_bf16_from_f32(const float* h, size_t n, std::vector& out_ptrs) { std::vector tmp(n); - for (size_t i = 0; i < n; ++i) + for (size_t i=0; i& wptrs, std::vector stacked(N_total * K); out_scales.clear(); int64_t row_off = 0; - for (size_t i = 0; i < wptrs.size(); ++i) { + for (size_t i=0; i tern; float sc; recover_ternary_and_scale(wptrs[i], Ns[i]*K, tern, sc); @@ -646,7 +646,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->vit_patch_b = mk_f32("vit.patch_embd.bias"); m->vit_pos = mk_f32("vit.pos_embd.weight"); m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { + for (int64_t i=0; ivit_layers && ok; ++i) { char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, s); return p; }; auto & w = m->vit[i]; w.ln1w=mk_f32(N("ln1.weight")); w.ln1b=mk_f32(N("ln1.bias")); @@ -669,7 +669,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->embed_tokens = m->packed_int2 ? nullptr : mk_mm("token_embd.weight"); m->lm_output_norm = mk_f32("lm.output_norm.weight"); m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { + for (int64_t i=0; ilm_layers && ok; ++i) { char p[64]; auto N = [&](const char * s) { std::snprintf(p, sizeof(p), "lm.blk.%lld.%s", (long long) i, s); return p; }; auto & w = m->lm[i]; w.attn_norm = mk_f32(N("attn_norm.weight")); @@ -762,7 +762,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, (int) m->lm_inter, (int) m->lm_layers, m->lm_rope_base, m->lm_rms_eps, max_seq); if (m->lm_cuda_ctx) { bool pack_ok = true; - for (int64_t L = 0; L < m->lm_layers && pack_ok && scales_ok; ++L) { + for (int64_t L=0; Llm_layers && pack_ok && scales_ok; ++L) { bitvla_lm_layer_cuda lyr{}; lyr.attn_norm_w = upload_bf16_from_f32((const float*) m->lm[L].attn_norm->data, m->lm_hidden, m->cuda_devptrs); @@ -862,7 +862,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, m->vit_ln_eps, mm_out); if (m->vit_cuda_ctx) { bool vit_ok = true; - for (int64_t L = 0; L < m->vit_layers && vit_ok; ++L) { + for (int64_t L=0; Lvit_layers && vit_ok; ++L) { bitvla_vit_layer_cuda vl{}; vl.ln1_w = upload_bf16_from_f32((const float*) m->vit[L].ln1w->data, m->vit_hidden, m->cuda_devptrs); @@ -916,7 +916,7 @@ std::unique_ptr bitvla_create(const std::string& mmproj_path, recover_ternary_and_scale(W, m->vit_hidden*m->vit_inter, tern, scale); std::vector padded((size_t) m->vit_hidden*ffn_pad, 0); - for (int64_t n = 0; n < m->vit_hidden; ++n) { + for (int64_t n=0; nvit_hidden; ++n) { std::memcpy(padded.data()+n * ffn_pad, tern.data()+n * m->vit_inter, (size_t) m->vit_inter); @@ -1111,7 +1111,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { img_embeds_host.assign((size_t) n_views * N * hidden_l, 0.0f); std::vector patches((size_t) N * patch_flat); const auto t_v0 = clk::now(); - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v BitvlaModelArch::predict(const Inputs& in) { return {}; } - for (int64_t pi = 0; pi < H/P; ++pi) - for (int64_t pj = 0; pj < H/P; ++pj) { + for (int64_t pi=0; pi BitvlaModelArch::predict(const Inputs& in) { if (cuda_vit_ready) { std::vector patches_bf16((size_t) N * patch_flat); - for (size_t i = 0; i < patches_bf16.size(); ++i) + for (size_t i=0; i BitvlaModelArch::predict(const Inputs& in) { std::vector img_bf16((size_t) N * hidden_l); cudaMemcpy(img_bf16.data(), d_vit_img_embeds, img_bf16.size()*sizeof(uint16_t), cudaMemcpyDeviceToHost); float* dst = img_embeds_host.data()+(size_t) v * N * hidden_l; - for (size_t i = 0; i < img_bf16.size(); ++i) { + for (size_t i=0; i BitvlaModelArch::predict(const Inputs& in) { ggml_tensor * pe = ggml_add(ctx, ggml_mul_mat(ctx, vit_patch_w, x_in), vit_patch_b); ggml_tensor * h = ggml_add(ctx, pe, vit_pos); - for (int64_t L = 0; L < vit_layers; ++L) { + for (int64_t L=0; L BitvlaModelArch::predict(const Inputs& in) { const int64_t n_action = num_actions_chunk * action_dim; int64_t n_image_markers = 0, n_proprio_markers = 0; - for (int64_t i = 0; i < n_lang_in; ++i) { + for (int64_t i=0; i BitvlaModelArch::predict(const Inputs& in) { if (!emb_reader.fetch_rows_f32("token_embd.weight", ids, inputs_embeds.data(), hidden_l)) return {}; int64_t k_img = 0; - for (int64_t i = 0; i < n_lang_in; ++i) { + for (int64_t i=0; i BitvlaModelArch::predict(const Inputs& in) { if (cuda_lm_ready && seq <= cuda_max_seq) { std::vector in_bf16((size_t) seq * hidden_l); - for (size_t i = 0; i < in_bf16.size(); ++i) + for (size_t i=0; i BitvlaModelArch::predict(const Inputs& in) { if (rc != 0) { std::fprintf(stderr, "vla(bitvla): CUDA LM forward failed\n"); return {}; } std::vector aids(n_action); - for (int64_t i = 0; i < n_action; ++i) + for (int64_t i=0; i out_bf16((size_t) n_action * hidden_l); cudaMemcpy(out_bf16.data(), d_action_hidden, out_bf16.size()*sizeof(uint16_t), cudaMemcpyDeviceToHost); - for (size_t i = 0; i < out_bf16.size(); ++i) { + for (size_t i=0; i BitvlaModelArch::predict(const Inputs& in) { ggml_set_name(positions, "positions"); ggml_tensor * h = x_in; - for (int64_t L = 0; L < lm_layers; ++L) { + for (int64_t L=0; L BitvlaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(x_in, inputs_embeds.data(), 0, ggml_nbytes(x_in)); std::vector pos_v(seq); - for (int64_t i = 0; i < seq; ++i) + for (int64_t i=0; i aids(n_action); - for (int64_t i = 0; i < n_action; ++i) + for (int64_t i=0; i BitvlaModelArch::predict(const Inputs& in) { " " + std::to_string(action_dim)); std::vector actions = std::move(normalized_actions); - for (int64_t t = 0; t < chunk; ++t) { - for (int64_t d = 0; d < action_dim; ++d) { + for (int64_t t=0; t & out) { const int64_t half = 128; const float lm = std::log(10000.0f); const float t = (float) bucket; out.assign(256, 0.0f); - for (int64_t i = 0; i < half; ++i) { + for (int64_t i=0; i & out) { inline void action_sinusoid(int64_t bucket, int64_t dim, int64_t T, std::vector & out) { const int64_t half = dim/2; const float step = std::log(10000.0f)/(float) half; const float t = (float) bucket; out.assign((size_t) T * dim, 0.0f); - for (int64_t tk = 0; tk < T; ++tk) for (int64_t i = 0; i < half; ++i) { + for (int64_t tk=0; tk sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { const int64_t half = dim/2; std::vector out(dim); - for (int64_t i = 0; i < half; ++i) { + for (int64_t i=0; i sinusoidal_time_emb(double t, int64_t dim, double min_ inline void build_causal_mask(int64_t seq, std::vector & out) { out.assign((size_t) seq * seq, 0.0f); const float NEG = -std::numeric_limits::infinity(); - for (int64_t q = 0; q < seq; ++q) - for (int64_t kv = q+1; kv < seq; ++kv) + for (int64_t q=0; q return false; } out.assign((size_t) 3*side * side, 0.0f); - for (int64_t h = 0; h < side; ++h) - for (int64_t w = 0; w < side; ++w) - for (int64_t c = 0; c < 3; ++c) { + for (int64_t h=0; hnb[1], x->nb[1])); @@ -418,7 +418,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, bool ok = true; m->lm_output_norm = mk_f32("vlm.output_norm.weight"); ok &= (m->lm_output_norm != nullptr); m->lm.resize(m->lm_layers); - for (int64_t i = 0; i < m->lm_layers && ok; ++i) { + for (int64_t i=0; ilm_layers && ok; ++i) { char p[64]; auto N = [&](const char * suf) { std::snprintf(p, sizeof(p), "vlm.blk.%lld.%s", (long long) i, suf); return p; }; auto & w = m->lm[i]; w.attn_norm = mk_f32(N("attn_norm.weight")); @@ -436,7 +436,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->ae_W3 = mk_mm("aex.ae.W3.weight"); m->ae_b3 = mk_f32("aex.ae.W3.bias"); m->ae_pos = mk_f32("aex.ae.pos_enc"); m->dit.resize(m->dit_layers); - for (int64_t i = 0; i < m->dit_layers && ok; ++i) { + for (int64_t i=0; idit_layers && ok; ++i) { char p[64]; auto N = [&](const char * suf) { std::snprintf(p, sizeof(p), "aex.blk.%lld.%s", (long long) i, suf); return p; }; auto & w = m->dit[i]; w.n1w = mk_f32(N("norm1.weight")); w.n1b = mk_f32(N("norm1.bias")); @@ -464,7 +464,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, m->vit_cls = mk_f32("vit.class_embd"); m->vit_pos = mk_f32("vit.pos_embd"); m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers && ok; ++i) { + for (int64_t i=0; ivit_layers && ok; ++i) { char p[64]; auto N = [&](const char * suf) { std::snprintf(p, sizeof(p), "vit.blk.%lld.%s", (long long) i, suf); return p; }; auto & w = m->vit[i]; w.n1w = mk_f32(N("norm1.weight")); w.n1b = mk_f32(N("norm1.bias")); @@ -546,14 +546,14 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { ggml_context * VC = vision_scratch.reset(vision_arena); if (!VC) { std::fprintf(stderr, "vla(evo1): ggml_init(vision ctx) failed\n"); return {}; } std::vector t_px((size_t) n_views), t_ie((size_t) n_views); - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v(n_views, 1), false); - for (int64_t v = 0; v < n_views; ++v) + for (int64_t v=0; v Evo1ModelArch::predict(const Inputs& in) { img_emb_host.assign((size_t) n_views * num_image_token * lm_hidden, 0.0f); std::vector chw; const auto tv0 = std::chrono::steady_clock::now(); - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v Evo1ModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(evo1): vision graph compute failed (%lld views)\n", (long long) n_views); return {}; } - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v Evo1ModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(evo1): note - %lld image views (model n_images=%lld); prompt adapts\n", (long long) n_views, (long long) n_images); bool pre_built = false; - for (int j = 0; j < in.n_lang; ++j) + for (int j=0; j Evo1ModelArch::predict(const Inputs& in) { std::vector input_ids; input_ids.reserve(max_text_length); if (pre_built) { - for (int j = 0; j < in.n_lang; ++j) + for (int j=0; j Evo1ModelArch::predict(const Inputs& in) { if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), lm_hidden)) return {}; { int64_t img_idx = 0; - for (int64_t p = 0; p < SEQ; ++p) { + for (int64_t p=0; p= n_img_tokens) { std::fprintf(stderr, "vla(evo1): more IMG_CTX tokens than ViT embeds\n"); return {}; } std::memcpy(inputs_embeds.data()+p * lm_hidden, @@ -639,15 +639,15 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { in.attention_mask_n, (long long) SEQ); return {}; } - for (int64_t p = 0; p < SEQ; ++p) + for (int64_t p=0; p state_norm(per_a, 0.0f); - for (int64_t i = 0; i < per_a; ++i) { + for (int64_t i=0; i Evo1ModelArch::predict(const Inputs& in) { const ggml_type at = act_type; ggml_tensor * h = as_type(C, t_embeds, at); - for (int64_t i = 0; i < lm_layers; ++i) + for (int64_t i=0; i Evo1ModelArch::predict(const Inputs& in) { struct DC { ggml_tensor *Wq, *bq, *K, *V; }; std::vector dc(dit_layers); - for (int64_t i = 0; i < dit_layers; ++i) { + for (int64_t i=0; i Evo1ModelArch::predict(const Inputs& in) { ae = ggml_add(C, ae, ae_pos); ae = ggml_relu(C, ggml_add(C, mm_act(C, ae_W2, ae, at), ae_b2)); ggml_tensor * x = ggml_add(C, mm_act(C, ae_W3, ae, at), ae_b3); - for (int64_t i = 0; i < dit_layers; ++i) { + for (int64_t i=0; i Evo1ModelArch::predict(const Inputs& in) { const float dt = 1.0f/(float) num_steps; ggml_tensor * x_action = t_x; - for (int64_t step = 0; step < num_steps; ++step) { + for (int64_t step=0; step Evo1ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_embeds, inputs_embeds.data(), 0, ggml_nbytes(t_embeds)); { std::vector pp(SEQ); - for (int64_t i = 0; i < SEQ; ++i) + for (int64_t i=0; i mk((size_t) SEQ * SEQ); const float NEG = -std::numeric_limits::infinity(); - for (int64_t q = 0; q < SEQ; ++q) for (int64_t kv = 0; kv < SEQ; ++kv) mk[q * SEQ+kv] = (kv <= q && attn_ok[kv]) ? 0.0f : NEG; + for (int64_t q=0; q am(per_a, 0.0f); for (int64_t i = 0; i < real_action_dim && i < per_a; ++i) am[i] = 1.0f; + { std::vector am(per_a, 0.0f); for (int64_t i=0; i qm(SEQ, 0.0f); for (int64_t p = 0; p < SEQ; ++p) qm[p] = attn_ok[p] ? 1.0f : 0.0f; + { std::vector qm(SEQ, 0.0f); for (int64_t p=0; p Evo1ModelArch::predict(const Inputs& in) { ggml_backend_tensor_get(x_action, x_final.data(), 0, x_final.size()*sizeof(float)); std::vector out((size_t) horizon * per_a); - for (int64_t hstep = 0; hstep < horizon; ++hstep) - for (int64_t c = 0; c < per_a; ++c) { + for (int64_t hstep=0; hstep Gr00tN1d5ModelArch::predict(const Inputs& in) { const auto tv0 = std::chrono::steady_clock::now(); std::vector chw; - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_tensor * state_features = aex.encode_state(C, t_state); std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); - for (int64_t i = 0; i < dit.cfg.layers; ++i) { + for (int64_t i=0; i Gr00tN1d5ModelArch::predict(const Inputs& in) { const float dt = 1.0f/(float) num_steps; ggml_tensor * actions = t_x0; - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); std::vector pp(SEQ); - for (int64_t i = 0; i < SEQ; ++i) + for (int64_t i=0; i Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); std::vector st(max_state_dim, 0.0f); - for (int64_t i = 0; i < max_state_dim; ++i) + for (int64_t i=0; i tau, tpr; action_sinusoid(bucket, E, AH, tau); diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index ec7816c..52f14fe 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -365,7 +365,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { std::vector shuf_host((size_t) c4*K*n_views); bool vok = true; - for (int64_t v = 0; v < n_views && vok; ++v) { + for (int64_t v=0; v Gr00tN1d6ModelArch::predict(const Inputs& in) { } if (vok) { ggml_backend_tensor_get(post_ln, post_ln_host.data(), 0, ggml_nbytes(post_ln)); - for (int64_t v = 0; v < n_views; ++v) + for (int64_t v=0; v Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_set_input(t_txt_idx); std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s Gr00tN1d6ModelArch::predict(const Inputs& in) { const int64_t every2 = 2*attend_text_every_n; std::vector Kc(dit.cfg.layers, nullptr), Vc(dit.cfg.layers, nullptr); - for (int64_t i = 0; i < dit.cfg.layers; ++i) { + for (int64_t i=0; i Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_embeds, inputs_embeds.data(), 0, ggml_nbytes(gio.t_embeds)); std::vector pp(SEQ); - for (int64_t i = 0; i < SEQ; ++i) + for (int64_t i=0; i Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_lmmask, mask.data(), 0, ggml_nbytes(gio.t_lmmask)); std::vector st(max_state_dim, 0.0f); - for (int64_t i = 0; i < max_state_dim; ++i) + for (int64_t i=0; i Gr00tN1d6ModelArch::predict(const Inputs& in) { if (gio.t_txt_idx) ggml_backend_tensor_set(gio.t_txt_idx, prompt.text_pos.data(), 0, ggml_nbytes(gio.t_txt_idx)); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s tau, tpr; action_sinusoid(bucket, E, AH, tau); diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index afdd8cb..5b34091 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -337,7 +337,7 @@ bool Gr00tN1d7ModelArch::build_caches() { interp_pos_embed(pos_table, num_side, vit_hidden, c_grow, c_gcol, grid, grid, c_pos_interp); c_tau.assign((size_t) num_steps, {}); c_tproj.assign((size_t) num_steps, {}); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s Gr00tN1d7ModelArch::predict(const Inputs& in) { if (in.precomputed_img_emb && in.n_img_views > 0) { n_views = in.n_img_views; img_emb_ptr = in.precomputed_img_emb; - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); } else if (in.images && in.n_images > 0) { n_views = in.n_images; img_emb_host.assign((size_t) n_views * K * H, 0.0f); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); ggml_context * VC = vision_scratch.reset((size_t) 512*1024*1024); @@ -388,15 +388,15 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_set_output(h); ggml_tensor * stash[3] = {nullptr, nullptr, nullptr}; - for (int64_t i = 0; i < vit_layers; ++i) { + for (int64_t i=0; i Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ggml_build_forward_expand(vg, ds_out[j]); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(gr00tn1d7): vision gallocr alloc failed\n"); return {}; } const auto tv0 = std::chrono::steady_clock::now(); std::vector patches; bool vok = true; - for (int64_t v = 0; v < n_views && vok; ++v) { + for (int64_t v=0; v Gr00tN1d7ModelArch::predict(const Inputs& in) { break; } ggml_backend_tensor_get(vit_embeds, img_emb_host.data()+v * K * H, 0, ggml_nbytes(vit_embeds)); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ggml_backend_tensor_get(ds_out[j], ds_host[j].data()+v * K * H, 0, ggml_nbytes(ds_out[j])); } stats.ms_vision = std::chrono::duration(std::chrono::steady_clock::now()-tv0).count(); @@ -439,16 +439,16 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector input_ids; int64_t n_img_slots = 0; - for (int j = 0; j < in.n_lang; ++j) + for (int j=0; j Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector inputs_embeds((size_t) SEQ * H); if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), H)) return {}; { int64_t k = 0; - for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { + for (int64_t p=0; p= n_img) { std::fprintf(stderr, "vla(gr00tn1d7): more tokens than ViT embeds\n"); return {}; } std::memcpy(inputs_embeds.data()+p * H, img_emb_ptr+k * H, H * sizeof(float)); ++k; } @@ -469,7 +469,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector image_pos_idx, text_pos_idx; image_pos_idx.reserve((size_t) n_img); text_pos_idx.reserve((size_t) (SEQ-n_img)); - for (int64_t p = 0; p < SEQ; ++p) { + for (int64_t p=0; p Gr00tN1d7ModelArch::predict(const Inputs& in) { std::vector> ds_pad(3); const bool inject_deepstack = (in.images && in.n_images > 0); - if (inject_deepstack) for (int j = 0; j < 3; ++j) { + if (inject_deepstack) for (int j=0; j<3; ++j) { ds_pad[j].assign((size_t) SEQ * H, 0.0f); - for (int64_t k = 0; k < n_img; ++k) { + for (int64_t k=0; k Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, max_state_dim, 1);ggml_set_input(t_state); ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); ggml_tensor * t_ds[3] = {nullptr,nullptr,nullptr}; - if (inject_deepstack) for (int j = 0; j < 3; ++j) { + if (inject_deepstack) for (int j=0; j<3; ++j) { t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_ds[j]); } @@ -529,13 +529,13 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { if (t_txt_idx) ggml_set_input(t_txt_idx); std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_set_output(vl_embs); vlsa_dump.push_back(vl_embs); } - for (int64_t i = 0; i < vlsa_layers; ++i) { + for (int64_t i=0; i Gr00tN1d7ModelArch::predict(const Inputs& in) { const int64_t every2 = 2*attend_text_every_n; std::vector Kc(dit_layers, nullptr), Vc(dit_layers, nullptr); - for (int64_t i = 0; i < dit_layers; ++i) { + for (int64_t i=0; i Gr00tN1d7ModelArch::predict(const Inputs& in) { } ggml_tensor * actions = t_x0; - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; snb[1], 0)); ggml_tensor * sa = ggml_concat(C, state_features, af, 1); ggml_tensor * hh = sa; - for (int64_t i = 0; i < dit_layers; ++i) { + for (int64_t i=0; i Gr00tN1d7ModelArch::predict(const Inputs& in) { int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; - for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { + for (int64_t i=st; i Gr00tN1d7ModelArch::predict(const Inputs& in) { } const int64_t this_t = n_img_tokens/(llm_grid_h * llm_grid_w); const int64_t image_offset = text_len+st_idx; - for (int64_t tt = 0; tt < this_t; ++tt) { - for (int64_t hh = 0; hh < llm_grid_h; ++hh) { - for (int64_t ww = 0; ww < llm_grid_w; ++ww) { + for (int64_t tt=0; tt Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_lmmask, c_mask.data(), 0, ggml_nbytes(t_lmmask)); { std::vector st(max_state_dim, 0.0f); - for (int64_t i = 0; i < max_state_dim; ++i) + for (int64_t i=0; i Gr00tN1d7ModelArch::predict(const Inputs& in) { dump_t("eagle", eagle); dump_t("vl_embs", vl_embs); - for (size_t li = 0; li < lm_h_dump.size(); ++li) { + for (size_t li=0; li Gr00tN1d7ModelArch::predict(const Inputs& in) { } }; - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v & q01, return false; if (!read_arr("mask", mk)) mk.assign(want, 1.0f); - mask.assign(mk.size(), 1); for (size_t i = 0; i < mk.size(); ++i) mask[i] = mk[i] != 0.0f ? 1 : 0; + mask.assign(mk.size(), 1); for (size_t i=0; i OpenVlaOftModelArch::predict(const Inputs& in) { if (n_views < 1) { std::fprintf(stderr, "vla(openvla_oft): need >=1 image view\n"); return {}; } if (!in.images) { std::fprintf(stderr, "vla(openvla_oft): n_images=%d but the images pointer is null\n", in.n_images); return {}; } // towers read S*S*3 per view; reject any view that is not exactly SxS. - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v OpenVlaOftModelArch::predict(const Inputs& in) { const int64_t L = in.n_lang; // ggml_get_rows does not bound-check, so reject out-of-range tokens here. - for (int64_t i = 0; i < L; ++i) + for (int64_t i=0; i= vocab) { std::fprintf(stderr, "vla(openvla_oft): token %d out of vocab\n", in.lang_tokens[i]); return {}; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 5e95ee7..f17e97f 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -494,7 +494,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); // patch embed (conv_2d) stays F32; the tower runs in the activation dtype ggml_tensor * h = as_type(VC, ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos), act_type); - for (int64_t i = 0; i < vit_layers; ++i) + for (int64_t i=0; i Pi0ModelArch::predict(const Inputs& in) { } const auto tv0 = clk::now(); std::vector chw; - for (int v = 0; v < in.n_images; ++v) { + for (int v=0; v Pi0ModelArch::predict(const Inputs& in) { ggml_tensor * t_suffix_pos= ggml_new_tensor_1d(C, GGML_TYPE_I32, n_suf); ggml_set_input(t_suffix_pos); ggml_tensor * t_full_mask = ggml_new_tensor_2d(C, GGML_TYPE_F32, n_total, n_suf); ggml_set_input(t_full_mask); std::vector t_time(num_steps); - for (int s = 0; s < num_steps; ++s) { + for (int s=0; s Pi0ModelArch::predict(const Inputs& in) { std::vector cK(n_layers), cV(n_layers); { ggml_tensor * h = prefix_embs; - for (int64_t i = 0; i < n_layers; ++i) { + for (int64_t i=0; i Pi0ModelArch::predict(const Inputs& in) { // do not accumulate in 8 mantissa bits. ggml_tensor * x_t = t_x0; std::vector v_steps(num_steps); - for (int step = 0; step < num_steps; ++step) { + for (int step=0; step Pi0ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_image_emb, img_emb_host.data(), 0, ggml_nbytes(t_image_emb)); ggml_backend_tensor_set(t_lang_emb, lang_rows.data(), 0, ggml_nbytes(t_lang_emb)); { - std::vector pp(n_prefix); for (int64_t i = 0; i < n_prefix; ++i) pp[i] = (int32_t) i; + std::vector pp(n_prefix); for (int64_t i=0; i sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix+i); + std::vector sp(n_suf); for (int64_t i=0; i sh(max_sd, 0.f); - for (int64_t i = 0; i < max_sd; ++i) + for (int64_t i=0; i Pi0ModelArch::predict(const Inputs& in) { { std::vector mk((size_t) n_total * n_suf); - for (int64_t i = 0; i < n_suf; ++i) - for (int64_t j = 0; j < n_total; ++j) { + for (int64_t i=0; i Pi0ModelArch::predict(const Inputs& in) { } ggml_backend_tensor_set(t_full_mask, mk.data(), 0, ggml_nbytes(t_full_mask)); } - for (int s = 0; s < num_steps; ++s) { + for (int s=0; s tv = sinusoidal_time_emb(timestep, hidden_ex, cfg.min_period, cfg.max_period); std::vector tile((size_t) hidden_ex * chunk); - for (int64_t c = 0; c < chunk; ++c) + for (int64_t c=0; c Pi0ModelArch::predict(const Inputs& in) { std::vector out((size_t) chunk * max_ad); ggml_backend_tensor_get(x_final, out.data(), 0, out.size()*sizeof(float)); - for (int64_t t = 0; t < chunk; ++t) { + for (int64_t t=0; t pi05_create(const std::string& mmproj_path, m->pl.declare(L, "vlm", cfg.n_layers, false); m->ex_layers.resize(cfg.n_layers); - for (int64_t i = 0; i < cfg.n_layers; ++i) { + for (int64_t i=0; iex_layers[i]; w.ada_in_w = L.f32 ("aex.blk.%lld.attn_norm.weight", (long long)i); w.ada_in_b = L.f32 ("aex.blk.%lld.attn_norm.bias", (long long)i); @@ -578,7 +578,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { ggml_tensor * conv = ggml_conv_2d(VC, vit.patch_w, t_px, (int) vit_patch_size, (int) vit_patch_size, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, grid * grid, vit_hidden))); ggml_tensor * h = ggml_add(VC, ggml_add(VC, patches, vit.patch_b), vit.pos); - for (int64_t i = 0; i < vit_layers; ++i) + for (int64_t i=0; i Pi05ModelArch::predict(const Inputs& in) { } const auto tv0 = clk::now(); std::vector chw; - for (int v = 0; v < in.n_images; ++v) { + for (int v=0; v Pi05ModelArch::predict(const Inputs& in) { ggml_tensor * t_suffix_pos= ggml_new_tensor_1d(C, GGML_TYPE_I32, n_suf); ggml_set_input(t_suffix_pos); std::vector t_time(num_steps); - for (int s = 0; s < num_steps; ++s) { + for (int s=0; s Pi05ModelArch::predict(const Inputs& in) { std::vector cK(n_layers), cV(n_layers); { ggml_tensor * h = prefix_embs; - for (int64_t i = 0; i < n_layers; ++i) { + for (int64_t i=0; i Pi05ModelArch::predict(const Inputs& in) { } ggml_tensor * x_t = t_x0; - for (int step = 0; step < num_steps; ++step) { + for (int step=0; step Pi05ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_image_emb, img_emb_host.data(), 0, ggml_nbytes(t_image_emb)); ggml_backend_tensor_set(t_lang_emb, lang_rows.data(), 0, ggml_nbytes(t_lang_emb)); { - std::vector pp(n_prefix); for (int64_t i = 0; i < n_prefix; ++i) pp[i] = (int32_t) i; + std::vector pp(n_prefix); for (int64_t i=0; i sp(n_suf); for (int64_t i = 0; i < n_suf; ++i) sp[i] = (int32_t) (n_prefix+i); + std::vector sp(n_suf); for (int64_t i=0; i Pi05ModelArch::predict(const Inputs& in) { } ggml_backend_tensor_set(t_x0, x0h.data(), 0, ggml_nbytes(t_x0)); } - for (int s = 0; s < num_steps; ++s) { + for (int s=0; s tv = sinusoidal_time_emb(timestep, hidden_ex, cfg.min_period, cfg.max_period); ggml_backend_tensor_set(t_time[s], tv.data(), 0, ggml_nbytes(t_time[s])); @@ -730,9 +730,9 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { ggml_backend_tensor_get(x_final, out.data(), 0, out.size()*sizeof(float)); if (!vla::env_flag("VLA_PI05_SKIP_UNNORM")) { - for (int64_t t = 0; t < chunk; ++t) { + for (int64_t t=0; t= cfg.real_action_dim) row[j] = 0.0f; else if (quantile_norm) diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index bc9b37b..4050b8d 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -71,7 +71,7 @@ struct safetensors { file.read(header_str.data(), header_size); data_blob_start = sizeof(uint64_t)+header_size; json j = json::parse(header_str); - for (auto it = j.begin(); it != j.end(); ++it) { + for (auto it=j.begin(); it!=j.end(); ++it) { if (it.key() == "__metadata__") continue; const auto & v = it.value(); @@ -187,12 +187,12 @@ struct gguf_source { const int nd_used = std::max(1, (int) pt_shape.size()); if (nd_used > GGML_MAX_DIMS) return false; - for (int d = 0; d < (int) pt_shape.size(); ++d) { + for (int d=0; d<(int) pt_shape.size(); ++d) { const int64_t expected = pt_shape[pt_shape.size()-1-d]; if (t->ne[d] != expected) return false; } - for (int d = (int) pt_shape.size(); d < GGML_MAX_DIMS; ++d) { + for (int d=(int) pt_shape.size(); dne[d] != 1) return false; } @@ -1132,7 +1132,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, pending_f32.push_back({std::string(VP) + "post_layernorm.weight", m->vit_post_ln_w, {H}}); pending_f32.push_back({std::string(VP) + "post_layernorm.bias", m->vit_post_ln_b, {H}}); m->vit.resize(m->vit_layers); - for (int64_t i = 0; i < m->vit_layers; ++i) { + for (int64_t i=0; ivit_layers; ++i) { EncBlockW & w = m->vit[i]; char pb[256]; std::snprintf(pb, sizeof(pb), "%sencoder.layers.%lld.", VP, (long long) i); const std::string pf = pb; @@ -1159,7 +1159,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, } m->vlm_layers.resize(cfg.n_layers); - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; ivlm_layers[i]; w.Wln_in = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.hidden); w.Wq = ggml_new_tensor_2d(ctx, wdt, cfg.hidden, cfg.q_full_dim); @@ -1189,7 +1189,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, m->Wnorm_vlm, {cfg.hidden}}); m->expert_layers.resize(cfg.n_layers); - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; iexpert_layers[i]; w.is_self_attn = (i%cfg.self_attn_every_n == 0); w.Wln_in = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, cfg.expert_h); @@ -1248,7 +1248,7 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, pending_f32.push_back({"model.action_out_proj.bias", m->b_aout, {cfg.max_action_dim}}); m->time_bcasts.assign(cfg.num_steps, nullptr); - for (int step = 0; step < cfg.num_steps; ++step) { + for (int step=0; steptime_bcasts[step] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cfg.expert_h, cfg.n_suffix); } @@ -1304,12 +1304,12 @@ SmolVLAModelArch* smolvla_load_impl(const std::string& mmproj_path, { const float dt = -1.f/static_cast(cfg.num_steps); - for (int step = 0; step < cfg.num_steps; ++step) { + for (int step=0; step tile(cfg.expert_h*cfg.n_suffix); - for (int64_t t = 0; t < cfg.n_suffix; ++t) { + for (int64_t t=0; t v_cache(cfg.n_layers); { ggml_tensor * h = prefix_embs; - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; ivlm_layers[i], h, mask_prefill_f16, pos_prefill, cfg_built, &k_cache[i], &v_cache[i]); } @@ -1396,7 +1396,7 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { // Reproject each cross-attn layer's prefix K/V once; reused every denoise step. std::vector xk_cache(cfg.n_layers, nullptr); std::vector xv_cache(cfg.n_layers, nullptr); - for (int li = 0; li < cfg.n_layers; ++li) { + for (int li=0; liexpert_layers[li].is_self_attn) expert_cross_kv(ctx, m->expert_layers[li], k_cache[li], v_cache[li], cfg_built, &xk_cache[li], &xv_cache[li]); @@ -1405,7 +1405,7 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { const float dt = -1.f/static_cast(cfg.num_steps); ggml_tensor * x_t = x0; - for (int step = 0; step < cfg.num_steps; ++step) { + for (int step=0; steptime_bcasts[step]; ggml_tensor * action_emb = ggml_add(ctx, ggml_mul_mat(ctx, m->W_ain, x_t), m->b_ain); ggml_tensor * action_time_in = ggml_concat(ctx, action_emb, time_bcast, 0); @@ -1414,7 +1414,7 @@ bool build_compute_graph(SmolVLAModelArch* m, int n_views) { ggml_mul_mat(ctx, m->W_at2, ggml_silu(ctx, mlp1)), m->b_at2); ggml_tensor * h = suffix_embs; - for (int li = 0; li < cfg.n_layers; ++li) { + for (int li=0; liexpert_layers[li].is_self_attn) { h = build_expert_self_attn_layer(ctx, m->expert_layers[li], h, k_cache[li], v_cache[li], @@ -1531,7 +1531,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_tensor * conv = ggml_conv_2d(VC, m->vit_patch_w, t_px, (int) m->vit_patch, (int) m->vit_patch, 0, 0, 1, 1); ggml_tensor * patches = ggml_cont(VC, ggml_transpose(VC, ggml_reshape_2d(VC, conv, n_patches, H))); ggml_tensor * hv = ggml_add(VC, ggml_add(VC, patches, m->vit_patch_b), m->vit_pos); - for (int64_t i = 0; i < m->vit_layers; ++i) + for (int64_t i=0; ivit_layers; ++i) hv = build_siglip_layer(VC, m->vit[i], hv, n_patches, m->vit_heads, H/m->vit_heads, H, m->vit_ln_eps); ggml_tensor * post_ln = ggml_add(VC, ggml_mul(VC, ggml_norm(VC, hv, m->vit_ln_eps), m->vit_post_ln_w), m->vit_post_ln_b); ggml_set_output(post_ln); @@ -1557,7 +1557,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector chw, post_host((size_t) H * n_patches), shuf_host((size_t) c4*K); bool vok = true; - for (int v = 0; v < n_views && vok; ++v) { + for (int v=0; vvit_image, chw)) { vok = false; break; @@ -1592,7 +1592,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { // its indices. Reject any token id outside the embedding table before the // gather so an out-of-range id cannot read past the weights. const int64_t vocab_rows = m->E_lang ? m->E_lang->ne[1] : 0; - for (int i = 0; i < in.n_lang; ++i) { + for (int i=0; i= vocab_rows) { std::fprintf(stderr, "vla: lang_tokens[%d]=%d out of vocab range [0, %lld)\n", i, in.lang_tokens[i], (long long) vocab_rows); @@ -1626,7 +1626,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector state_host(cfg.max_state_dim, 0.0f); if (in.state) std::memcpy(state_host.data(), in.state, cfg.max_state_dim*sizeof(float)); - for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { + for (int64_t i=0; istate_mean[i])/(m->state_std[i]+cfg.norm_eps); } @@ -1646,8 +1646,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { const int64_t suffix_pos_base = state_pos+1; std::vector mask_prefill_host(n_prefix_max * n_prefix_max); std::vector pos_prefill_host (n_prefix_max); - for (int64_t i = 0; i < n_prefix_max; ++i) { - for (int64_t j = 0; j < n_prefix_max; ++j) { + for (int64_t i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector mask_prefix_only_host(n_prefix_max * cfg.n_suffix, 0.f); std::vector pos_full_host (cfg.n_suffix); std::vector pos_rebased_host (cfg.n_suffix); - for (int64_t i = 0; i < cfg.n_suffix; ++i) { - for (int64_t j = 0; j < n_full_max; ++j) { + for (int64_t i=0; i= pad_start && j < pad_end); @@ -1674,7 +1674,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { } mask_full_host[i * n_full_max+j] = blocked ? -INFINITY : 0.f; } - for (int64_t j = 0; j < n_prefix_max; ++j) { + for (int64_t j=0; j= pad_start && j < pad_end) { mask_prefix_only_host[i * n_prefix_max+j] = -INFINITY; } @@ -1709,9 +1709,9 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector out(cfg.n_suffix*cfg.max_action_dim); ggml_backend_tensor_get(m->out_x_t, out.data(), 0, out.size()*sizeof(float)); - for (int64_t r = 0; r < cfg.n_suffix; ++r) { + for (int64_t r=0; raction_std[j]+cfg.norm_eps)+m->action_mean[j] : 0.0f; } } @@ -1757,7 +1757,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector state_host(cfg.max_state_dim, 0.0f); if (in.state) std::memcpy(state_host.data(), in.state, cfg.max_state_dim*sizeof(float)); - for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) { + for (int64_t i=0; istate_mean[i])/(m->state_std[i]+cfg.norm_eps); } @@ -1772,8 +1772,8 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector mask_prefill_host(cfg.n_prefix*cfg.n_prefix); std::vector pos_prefill_host (cfg.n_prefix); - for (int64_t i = 0; i < cfg.n_prefix; ++i) { - for (int64_t j = 0; j < cfg.n_prefix; ++j) { + for (int64_t i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector mask_prefix_only_host(cfg.n_prefix*cfg.n_suffix, 0.f); std::vector pos_full_host (cfg.n_suffix); std::vector pos_rebased_host (cfg.n_suffix); - for (int64_t i = 0; i < cfg.n_suffix; ++i) { - for (int64_t j = 0; j < cfg.n_full; ++j) { + for (int64_t i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector v_cache(cfg.n_layers); { ggml_tensor * h = prefix_embs; - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; ivlm_layers[i], h, mask_prefill_f16, pos_prefill, cfg, &k_cache[i], &v_cache[i]); } @@ -1822,7 +1822,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector K_storage(cfg.n_layers); std::vector V_storage(cfg.n_layers); if (in.timing_detail == TimingDetail::PHASE) { - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector xk_cache(cfg.n_layers, nullptr); std::vector xv_cache(cfg.n_layers, nullptr); - for (int li = 0; li < cfg.n_layers; ++li) { + for (int li=0; liexpert_layers[li].is_self_attn) expert_cross_kv(ctx, m->expert_layers[li], K_ref[li], V_ref[li], cfg, &xk_cache[li], &xv_cache[li]); } - for (int step = 0; step < cfg.num_steps; ++step) { + for (int step=0; step(step)*static_cast(dt); time_host[step] = sinusoidal_time_emb(time, cfg.expert_h, cfg.min_period, cfg.max_period); @@ -1864,7 +1864,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { m->b_at2); ggml_tensor * h = suffix_embs; - for (int li = 0; li < cfg.n_layers; ++li) { + for (int li=0; liexpert_layers[li].is_self_attn) { h = build_expert_self_attn_layer(ctx, m->expert_layers[li], h, K_ref[li], V_ref[li], @@ -1897,10 +1897,10 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_backend_tensor_set(mask_prefix_only, mask_prefix_only_host.data(), 0, mask_prefix_only_host.size()*sizeof(float)); ggml_backend_tensor_set(pos_full, pos_full_host.data(), 0, pos_full_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(pos_rebased, pos_rebased_host.data(), 0, pos_rebased_host.size() * sizeof(int32_t)); - for (int step = 0; step < cfg.num_steps; ++step) { + for (int step=0; step tile(cfg.expert_h*cfg.n_suffix); - for (int64_t t = 0; t < cfg.n_suffix; ++t) { + for (int64_t t=0; t predict_impl(SmolVLAModelArch* m, const Inputs& in) { if (in.timing_detail == TimingDetail::PHASE) { ggml_cgraph * gf_pre = ggml_new_graph_custom(ctx, 4096, false); - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { } m->stats.ms_prefill = ms_since(t0); - for (int i = 0; i < cfg.n_layers; ++i) { + for (int i=0; i predict_impl(SmolVLAModelArch* m, const Inputs& in) { std::vector out(cfg.n_suffix*cfg.max_action_dim); ggml_backend_tensor_get(x_t, out.data(), 0, out.size()*sizeof(float)); - for (int64_t r = 0; r < cfg.n_suffix; ++r) { + for (int64_t r=0; raction_std[j]+cfg.norm_eps)+m->action_mean[j] : 0.0f; } } diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index aae2610..65c9e2c 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -96,7 +96,7 @@ bool parse_stats(const std::string & js, int64_t want, std::vector & q01, return false; if (!read_arr("mask", mk)) mk.assign(want, 1.0f); - mask.assign(mk.size(), 1); for (size_t i = 0; i < mk.size(); ++i) mask[i] = mk[i] != 0.0f ? 1 : 0; + mask.assign(mk.size(), 1); for (size_t i=0; i VlaAdapterModelArch::predict(const Inputs& in) { if (n_views < 1) { std::fprintf(stderr, "vla(vla_adapter): need >=1 image view\n"); return {}; } if (!in.images) { std::fprintf(stderr, "vla(vla_adapter): n_images=%d but the images pointer is null\n", in.n_images); return {}; } // towers read S*S*3 per view; reject any view that is not exactly SxS. - for (int64_t v = 0; v < n_views; ++v) { + for (int64_t v=0; v VlaAdapterModelArch::predict(const Inputs& in) { const int64_t NPROMPT = in.n_lang; // ggml_get_rows does not bound-check, so reject out-of-range tokens here. - for (int64_t i = 0; i < NPROMPT; ++i) + for (int64_t i=0; i= vocab) { std::fprintf(stderr, "vla(vla_adapter): token %d out of vocab\n", in.lang_tokens[i]); return {}; diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index be6dd00..27bc6a0 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -306,7 +306,7 @@ bool VlaJepaModelArch::build_caches() { interp_pos_embed(pos_table, num_side, vit_hidden, c_grow, c_gcol, grid, grid, c_pos_interp); c_tau.assign((size_t) num_steps, {}); c_tproj.assign((size_t) num_steps, {}); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s VlaJepaModelArch::predict(const Inputs& in) { int64_t n_views = in.n_images; if (n_views <= 0) { std::fprintf(stderr, "vla(vla_jepa): no images in the request\n"); return {}; } std::vector img_emb_host((size_t) n_views * K * H), ds_host[3]; - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ds_host[j].assign((size_t) n_views * K * H, 0.0f); std::vector inj_patches; const char * patches_file = std::getenv("VLA_JEPA_PATCHES"); @@ -391,15 +391,15 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * h = ggml_add(VC, ggml_add(VC, ggml_mul_mat(VC, vit.patch_w, t_patches), vit.patch_b), t_pos); ggml_set_output(h); ggml_tensor * stash[3] = {nullptr, nullptr, nullptr}; - for (int64_t i = 0; i < vit_layers; ++i) { + for (int64_t i=0; i VlaJepaModelArch::predict(const Inputs& in) { ggml_set_output(vit_embeds); ggml_cgraph * vg = ggml_new_graph_custom(VC, 16384, false); ggml_build_forward_expand(vg, vit_embeds); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ggml_build_forward_expand(vg, ds_out[j]); if (!vision_scratch.alloc(backend, vg)) { std::fprintf(stderr, "vla(vla_jepa): vision gallocr alloc failed\n"); return {}; } const auto tv0 = std::chrono::steady_clock::now(); std::vector patches; bool vok = true; - for (int64_t v = 0; v < n_views && vok; ++v) { + for (int64_t v=0; v VlaJepaModelArch::predict(const Inputs& in) { break; } ggml_backend_tensor_get(vit_embeds, img_emb_host.data()+v * K * H, 0, ggml_nbytes(vit_embeds)); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ggml_backend_tensor_get(ds_out[j], ds_host[j].data()+v * K * H, 0, ggml_nbytes(ds_out[j])); if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data()+v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } } @@ -442,16 +442,16 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { std::vector input_ids; int64_t n_img_slots = 0; - for (int j = 0; j < in.n_lang; ++j) + for (int j=0; j VlaJepaModelArch::predict(const Inputs& in) { std::vector inputs_embeds((size_t) SEQ * H); if (!io.fetch_rows_f32("token_embd.weight", input_ids, inputs_embeds.data(), H)) return {}; - { int64_t k = 0; for (int64_t p = 0; p < SEQ; ++p) if (input_ids[p] == (int32_t) image_token_index) { std::memcpy(inputs_embeds.data()+p * H, img_emb_host.data()+k * H, H * sizeof(float)); ++k; } } + { int64_t k = 0; for (int64_t p=0; p image_pos_idx, emb_pos_idx; - for (int64_t p = 0; p < SEQ; ++p) { + for (int64_t p=0; p VlaJepaModelArch::predict(const Inputs& in) { if ((int64_t) emb_pos_idx.size() != num_future) { std::fprintf(stderr, "vla(vla_jepa): found %zu embodied tokens, expected %lld\n", emb_pos_idx.size(), (long long) num_future); return {}; } std::vector> ds_pad(3); - for (int j = 0; j < 3; ++j) { + for (int j=0; j<3; ++j) { ds_pad[j].assign((size_t) SEQ * H, 0.0f); - for (int64_t k = 0; k < n_img; ++k) + for (int64_t k=0; k VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * t_lmmask = ggml_new_tensor_2d(C, GGML_TYPE_F32, SEQ, SEQ); ggml_set_input(t_lmmask); ggml_tensor * t_emb_idx= ggml_new_tensor_1d(C, GGML_TYPE_I32, num_future); ggml_set_input(t_emb_idx); ggml_tensor * t_ds[3]; - for (int j = 0; j < 3; ++j) { + for (int j=0; j<3; ++j) { t_ds[j] = ggml_new_tensor_2d(C, GGML_TYPE_F32, H, SEQ); ggml_set_input(t_ds[j]); } ggml_tensor * hh = t_embeds; - for (int64_t i = 0; i < lm_layers; ++i) { + for (int64_t i=0; i VlaJepaModelArch::predict(const Inputs& in) { int64_t st = 0, st_idx = 0; while (st < SEQ) { int64_t img_start = -1; - for (int64_t i = st; i < SEQ; ++i) if (input_ids[i] == (int32_t) image_token_index) { + for (int64_t i=st; i VlaJepaModelArch::predict(const Inputs& in) { const int64_t n_img_tokens = img_end-img_start; const int64_t this_t = n_img_tokens/(llm_grid * llm_grid); const int64_t image_offset = text_len+st_idx; - for (int64_t tt = 0; tt < this_t; ++tt) for (int64_t hy = 0; hy < llm_grid; ++hy) for (int64_t wx = 0; wx < llm_grid; ++wx) { + for (int64_t tt=0; tt VlaJepaModelArch::predict(const Inputs& in) { } ggml_backend_tensor_set(t_lmmask, c_mask.data(), 0, ggml_nbytes(t_lmmask)); ggml_backend_tensor_set(t_emb_idx, emb_pos_idx.data(), 0, ggml_nbytes(t_emb_idx)); - for (int j = 0; j < 3; ++j) + for (int j=0; j<3; ++j) ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); const auto tp0 = std::chrono::steady_clock::now(); @@ -585,7 +585,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * t_state = ggml_new_tensor_2d(C, GGML_TYPE_F32, state_dim, 1); ggml_set_input(t_state); ggml_tensor * t_x0 = ggml_new_tensor_2d(C, GGML_TYPE_F32, AD, AH); ggml_set_input(t_x0); std::vector t_tau(num_steps), t_tproj(num_steps); - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s VlaJepaModelArch::predict(const Inputs& in) { step_vel.assign(num_steps, nullptr); step_act.assign(num_steps, nullptr); ggml_tensor * actions = t_x0; - for (int64_t s = 0; s < num_steps; ++s) { + for (int64_t s=0; s VlaJepaModelArch::predict(const Inputs& in) { ggml_tensor * seq = ggml_concat(C, ggml_concat(C, state_features, future, 1), af, 1); step_seq[s] = seq; ggml_tensor * x = seq; - for (int64_t i = 0; i < dit_layers; ++i) { + for (int64_t i=0; i VlaJepaModelArch::predict(const Inputs& in) { ggml_cgraph * hg = ggml_new_graph_custom(C, 65536, false); ggml_build_forward_expand(hg, actions); - if (dump_prefix) for (int64_t s = 0; s < num_steps; ++s) { + if (dump_prefix) for (int64_t s=0; s VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_cond, cond_host.data(), 0, ggml_nbytes(t_cond)); { std::vector st(state_dim, 0.0f); - for (int64_t i = 0; i < state_dim; ++i) + for (int64_t i=0; i VlaJepaModelArch::predict(const Inputs& in) { stats.ms_denoise = std::chrono::duration(std::chrono::steady_clock::now()-td0).count(); stats.ms_inference = stats.ms_prefill+stats.ms_denoise; - if (dump_prefix) for (int64_t s = 0; s < num_steps; ++s) { + if (dump_prefix) for (int64_t s=0; s & row, std::vector & col) { const int64_t S = gh * gw; row.assign(S, 0); col.assign(S, 0); - for (int64_t s = 0; s < S; ++s) { + for (int64_t s=0; s & row, const std::vector< std::vector & cos_t, std::vector & sin_t) { const int64_t S = (int64_t) row.size(), nf = hd/4; std::vector invf(nf); - for (int64_t i = 0; i < nf; ++i) + for (int64_t i=0; i emb(hd); - for (int64_t i = 0; i < nf; ++i) { + for (int64_t i=0; i & table, int64_t num_side, const int64_t S = (int64_t) row.size(); out.assign((size_t) S * hidden, 0.0f); auto src_coord = [&](int64_t k, int64_t g) -> double { return (g <= 1) ? 0.0 : (double) k * (double)(num_side-1)/(double)(g-1); }; - for (int64_t s = 0; s < S; ++s) { + for (int64_t s=0; s & table, int64_t num_side, const double c00 = (1-dh)*(1-dw), c01 = (1-dh)*dw, c10 = dh * (1-dw), c11 = dh * dw; const float * T00 = &table[(h0*num_side+w0)*hidden]; const float * T01 = &table[(h0*num_side+w1)*hidden]; const float * T10 = &table[(h1*num_side+w0)*hidden]; const float * T11 = &table[(h1*num_side+w1)*hidden]; - for (int64_t c = 0; c < hidden; ++c) + for (int64_t c=0; c positionals; - for (int i = 1; i < argc; ++i) { + for (int i=1; i const char * { if (i+1 >= argc) { @@ -126,26 +126,26 @@ int main(int argc, char ** argv) { std::vector> pixels(n_images, std::vector((size_t) 3*side * side)); std::vector views(n_images); - for (int v = 0; v < n_images; ++v) { - for (int y = 0; y < side; ++y) - for (int x = 0; x < side; ++x) - for (int c = 0; c < 3; ++c) + for (int v=0; v lang((size_t) n_tokens); - for (int i = 0; i < n_tokens; ++i) + for (int i=0; i= 0 && extra_count > 0) lang.insert(lang.end(), (size_t) extra_count, extra_token); std::vector state((size_t) cfg.max_state_dim, 0.0f); - for (int64_t i = 0; i < cfg.real_state_dim && i < cfg.max_state_dim; ++i) + for (int64_t i=0; i noise((size_t) cfg.max_action_dim*(size_t) cfg.n_suffix); - for (size_t i = 0; i < noise.size(); ++i) + for (size_t i=0; i ms; ms.reserve((size_t) reps); double vision_sum = 0.0; - for (int i = 0; i < reps; ++i) { + for (int i=0; i out = vla::predict(m, in); const auto t1 = std::chrono::steady_clock::now(); diff --git a/src/serving/vla-cli.cpp b/src/serving/vla-cli.cpp index 81156d2..67c9c23 100644 --- a/src/serving/vla-cli.cpp +++ b/src/serving/vla-cli.cpp @@ -212,7 +212,7 @@ int main(int argc, char ** argv) { std::vector image_paths; bool pretty = false; - for (int i = 1; i < argc; ++i) { + for (int i=1; i const char * { if (i+1 >= argc) { @@ -288,7 +288,7 @@ int main(int argc, char ** argv) { std::vector> imgbuf(image_paths.size()); std::vector views(image_paths.size()); - for (size_t v = 0; v < image_paths.size(); ++v) { + for (size_t v=0; v 0 ? cfg.max_action_dim : 1; if (pretty) { - for (size_t i = 0; i < act.size(); ++i) + for (size_t i=0; i positionals; - for (int i = 1; i < argc; ++i) { + for (int i=1; i images; images.reserve(req.images_size()); bool decode_ok = true; - for (int v = 0; v < req.images_size(); ++v) { + for (int v=0; v views((size_t) (in->n_images > 0 ? in->n_images : 0)); - for (size_t i = 0; i < views.size(); ++i) { + for (size_t i=0; iimages[i].data, in->images[i].w, in->images[i].h, diff --git a/src/vlm/engine.cpp b/src/vlm/engine.cpp index 42dcd64..e79c147 100644 --- a/src/vlm/engine.cpp +++ b/src/vlm/engine.cpp @@ -180,7 +180,7 @@ ChatResult Engine::chat(const std::vector & messages, } int img_msg_idx = -1; - for (int i = (int) messages.size()-1; i >= 0; --i) { + for (int i=(int) messages.size()-1; i>=0; --i) { if (messages[i].role == "user") { img_msg_idx = i; break; @@ -193,7 +193,7 @@ ChatResult Engine::chat(const std::vector & messages, const char * marker = mtmd_default_marker(); const int64_t t_prefill_start = ggml_time_us(); - for (size_t i = 0; i < messages.size(); ++i) { + for (size_t i=0; i & messages, if (msg.content.find(marker) == std::string::npos) { std::string prefix; - for (size_t k = 0; k < images.size(); ++k) + for (size_t k=0; k & messages, text.parse_special = true; std::vector bmp_ptrs(bmps.size()); - for (size_t k = 0; k < bmps.size(); ++k) + for (size_t k=0; k & messages, std::vector generated; const int64_t t_decode_start = ggml_time_us(); res.finish_reason = "length"; - for (int i = 0; i < n_predict; ++i) { + for (int i=0; ilctx, -1); common_sampler_accept(smpl, tok, true); generated.push_back(tok); From 04df9618d5723d41b2a3cbce10aafe252258da1d Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Fri, 14 Aug 2026 01:28:05 +0700 Subject: [PATCH 18/21] move the eval and ci harnesses onto the runtime option flags --- ci/lib/common.sh | 2 +- docs/backend/sycl.md | 2 +- .../corl-paper/experiments/compile_compare.md | 406 + .../experiments/ur10e-vlacpp-smolvla-mac.md | 317 + .../ur10e-vlacpp-smolvla-ryzen7.md | 467 + .../experiments/ur10e-vlacpp-smolvla.md | 441 + docs/corl-paper/solver_sweep.pdf | Bin 0 -> 20987 bytes docs/corl-paper/solver_sweep.png | Bin 0 -> 104785 bytes eval/README.md | 4 +- eval/bitvla_ref/bench_bitvla_pytorch.py | 317 + eval/bitvla_ref/instrument.py | 291 + eval/bitvla_ref/run_libero_bitvla_pytorch.py | 220 + eval/bitvla_ref/verify_compile_parity.py | 205 + eval/collect_latency_compare.py | 450 + eval/collect_libero_per_model.py | 333 + eval/collect_solver_displacement.py | 134 + eval/collect_solver_sweep.py | 197 + eval/collect_sr_compare.py | 235 + eval/compare_act_dtype.py | 175 + eval/plot_solver_sweep.py | 210 + eval/pytorch_ref/client/__init__.py | 0 eval/pytorch_ref/client/run_libero_eval.py | 152 + eval/pytorch_ref/policies/__init__.py | 13 + eval/pytorch_ref/policies/action_chunk.py | 50 + eval/pytorch_ref/policies/evo1/__init__.py | 139 + .../evo1/model/action_head/__init__.py | 0 .../evo1/model/action_head/flow_matching.py | 458 + .../policies/evo1/model/internvl3/__init__.py | 0 .../model/internvl3/internvl3_embedder.py | 261 + eval/pytorch_ref/policies/evo1/normalizer.py | 49 + eval/pytorch_ref/policies/evo1/policy.py | 182 + .../policies/gr00t_n15/__init__.py | 125 + .../policies/gr00t_n15/groot/__init__.py | 4 + .../gr00t_n15/groot/action_head/__init__.py | 14 + .../groot/action_head/action_encoder.py | 54 + .../groot/action_head/cross_attention_dit.py | 370 + .../action_head/flow_matching_action_head.py | 406 + .../gr00t_n15/groot/configuration_groot.py | 202 + .../configuration_eagle2_5_vl.py | 135 + .../image_processing_eagle2_5_vl_fast.py | 504 + .../eagle2_hg_model/modeling_eagle2_5_vl.py | 395 + .../eagle2_hg_model/processing_eagle2_5_vl.py | 518 + .../policies/gr00t_n15/groot/groot_n1.py | 376 + .../gr00t_n15/groot/modeling_groot.py | 332 + .../policies/gr00t_n15/groot/utils.py | 47 + .../policies/gr00t_n16/__init__.py | 408 + .../policies/gr00t_n16/data/__init__.py | 0 .../policies/gr00t_n16/data/data_config.py | 79 + .../gr00t_n16/data/embodiment_configs.py | 355 + .../gr00t_n16/data/embodiment_tags.py | 61 + .../policies/gr00t_n16/data/interfaces.py | 128 + .../data/state_action/action_chunking.py | 666 + .../gr00t_n16/data/state_action/pose.py | 711 + .../state_action/state_action_processor.py | 665 + .../policies/gr00t_n16/data/types.py | 103 + .../policies/gr00t_n16/data/utils.py | 292 + .../policies/gr00t_n16/model/__init__.py | 2 + .../gr00t_n16/model/gr00t_n1d6/gr00t_n1d6.py | 698 + .../model/gr00t_n1d6/image_augmentations.py | 575 + .../model/gr00t_n1d6/processing_gr00t_n1d6.py | 542 + .../gr00t_n16/model/modules/__init__.py | 1 + .../policies/gr00t_n16/model/modules/dit.py | 468 + .../gr00t_n16/model/modules/eagle_backbone.py | 126 + .../modules/embodiment_conditioned_mlp.py | 223 + .../model/modules/flowmatching_modules.py | 98 + .../Eagle-Block2A-2B-v2/added_tokens.json | 39 + .../Eagle-Block2A-2B-v2/chat_template.json | 3 + .../nvidia/Eagle-Block2A-2B-v2/config.json | 84 + .../configuration_eagle3_vl.py | 98 + .../generation_config.json | 6 + .../image_processing_eagle3_vl_fast.py | 233 + .../nvidia/Eagle-Block2A-2B-v2/merges.txt | 151388 ++++++++++++++ .../Eagle-Block2A-2B-v2/modeling_eagle3_vl.py | 457 + .../Eagle-Block2A-2B-v2/modeling_siglip2.py | 1479 + .../preprocessor_config.json | 36 + .../processing_eagle3_vl.py | 972 + .../Eagle-Block2A-2B-v2/processor_config.json | 14 + .../special_tokens_map.json | 42 + .../Eagle-Block2A-2B-v2/tokenizer_config.json | 344 + .../nvidia/Eagle-Block2A-2B-v2/vocab.json | 151645 +++++++++++++++ .../policies/gr00t_n17/__init__.py | 360 + .../policies/gr00t_n17/data/__init__.py | 0 .../gr00t_n17/data/embodiment_configs.py | 256 + .../gr00t_n17/data/embodiment_tags.py | 208 + .../policies/gr00t_n17/data/interfaces.py | 142 + .../gr00t_n17/data/state_action/__init__.py | 0 .../data/state_action/action_chunking.py | 680 + .../gr00t_n17/data/state_action/pose.py | 726 + .../state_action/state_action_processor.py | 677 + .../policies/gr00t_n17/data/types.py | 126 + .../policies/gr00t_n17/data/utils.py | 307 + .../policies/gr00t_n17/model/__init__.py | 2 + .../gr00t_n17/model/gr00t_n1d7/__init__.py | 0 .../gr00t_n17/model/gr00t_n1d7/gr00t_n1d7.py | 768 + .../model/gr00t_n1d7/image_augmentations.py | 591 + .../model/gr00t_n1d7/processing_gr00t_n1d7.py | 784 + .../gr00t_n17/model/modules/__init__.py | 1 + .../policies/gr00t_n17/model/modules/dit.py | 483 + .../modules/embodiment_conditioned_mlp.py | 238 + .../model/modules/flowmatching_modules.py | 113 + .../gr00t_n17/model/modules/qwen3_backbone.py | 153 + eval/pytorch_ref/policies/pi0.py | 147 + eval/pytorch_ref/policies/smolvla.py | 73 + eval/pytorch_ref/policies/torch_compile.py | 192 + .../policies/vla_adapter/__init__.py | 94 + .../pytorch_ref/policies/vla_adapter/model.py | 334 + .../policies/vla_adapter/verify_ref.py | 73 + eval/pytorch_ref/pyproject.toml | 67 + eval/pytorch_ref/server/evo1_server.py | 42 + eval/pytorch_ref/server/gr00t_n15_server.py | 42 + eval/pytorch_ref/server/gr00t_n17_server.py | 51 + eval/pytorch_ref/server/gr00t_server.py | 51 + eval/pytorch_ref/server/pi0_server.py | 42 + eval/pytorch_ref/server/smolvla_server.py | 42 + eval/pytorch_ref/sim | 1 + eval/pytorch_ref/utils/service.py | 300 + .../utils/sim_adapters/__init__.py | 0 eval/pytorch_ref/utils/sim_adapters/base.py | 16 + eval/pytorch_ref/utils/sim_adapters/libero.py | 141 + .../pytorch_ref/utils/sim_adapters/simpler.py | 40 + eval/pytorch_ref/uv.lock | 5408 + eval/run_bitvla_compile_compare.sh | 128 + eval/run_bitvla_pytorch_perf.sh | 133 + eval/run_latency_compare.sh | 390 + eval/run_latency_compare_all.sh | 62 + eval/run_latency_serial.sh | 76 + eval/run_libero.sh | 5 +- eval/run_libero_bitvla.sh | 332 + eval/run_libero_client.sh | 5 +- eval/run_libero_compare.sh | 185 + eval/run_libero_pytorch.sh | 284 + eval/run_libero_server.sh | 10 +- eval/run_simpler.sh | 4 +- eval/run_solver_step_displacement.sh | 69 + eval/run_solver_step_sweep.sh | 156 + eval/setup_bitvla_ref_env.sh | 108 + 136 files changed, 337932 insertions(+), 19 deletions(-) create mode 100644 docs/corl-paper/experiments/compile_compare.md create mode 100644 docs/corl-paper/experiments/ur10e-vlacpp-smolvla-mac.md create mode 100644 docs/corl-paper/experiments/ur10e-vlacpp-smolvla-ryzen7.md create mode 100644 docs/corl-paper/experiments/ur10e-vlacpp-smolvla.md create mode 100644 docs/corl-paper/solver_sweep.pdf create mode 100644 docs/corl-paper/solver_sweep.png create mode 100644 eval/bitvla_ref/bench_bitvla_pytorch.py create mode 100644 eval/bitvla_ref/instrument.py create mode 100644 eval/bitvla_ref/run_libero_bitvla_pytorch.py create mode 100644 eval/bitvla_ref/verify_compile_parity.py create mode 100644 eval/collect_latency_compare.py create mode 100755 eval/collect_libero_per_model.py create mode 100755 eval/collect_solver_displacement.py create mode 100644 eval/collect_solver_sweep.py create mode 100644 eval/collect_sr_compare.py create mode 100644 eval/compare_act_dtype.py create mode 100644 eval/plot_solver_sweep.py create mode 100644 eval/pytorch_ref/client/__init__.py create mode 100644 eval/pytorch_ref/client/run_libero_eval.py create mode 100644 eval/pytorch_ref/policies/__init__.py create mode 100644 eval/pytorch_ref/policies/action_chunk.py create mode 100644 eval/pytorch_ref/policies/evo1/__init__.py create mode 100644 eval/pytorch_ref/policies/evo1/model/action_head/__init__.py create mode 100755 eval/pytorch_ref/policies/evo1/model/action_head/flow_matching.py create mode 100644 eval/pytorch_ref/policies/evo1/model/internvl3/__init__.py create mode 100644 eval/pytorch_ref/policies/evo1/model/internvl3/internvl3_embedder.py create mode 100644 eval/pytorch_ref/policies/evo1/normalizer.py create mode 100644 eval/pytorch_ref/policies/evo1/policy.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/action_head/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/action_head/action_encoder.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/action_head/cross_attention_dit.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/action_head/flow_matching_action_head.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/configuration_groot.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/configuration_eagle2_5_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/image_processing_eagle2_5_vl_fast.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/modeling_eagle2_5_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/processing_eagle2_5_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/groot_n1.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/modeling_groot.py create mode 100644 eval/pytorch_ref/policies/gr00t_n15/groot/utils.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/__init__.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/data/data_config.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/embodiment_configs.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/data/embodiment_tags.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/interfaces.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/state_action/action_chunking.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/state_action/pose.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/state_action/state_action_processor.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/types.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/data/utils.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/__init__.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/gr00t_n1d6/gr00t_n1d6.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/gr00t_n1d6/image_augmentations.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/gr00t_n1d6/processing_gr00t_n1d6.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/__init__.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/modules/dit.py create mode 100755 eval/pytorch_ref/policies/gr00t_n16/model/modules/eagle_backbone.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/embodiment_conditioned_mlp.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/flowmatching_modules.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/added_tokens.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/chat_template.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/config.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/configuration_eagle3_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/generation_config.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/image_processing_eagle3_vl_fast.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/merges.txt create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/modeling_eagle3_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/modeling_siglip2.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/preprocessor_config.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/processing_eagle3_vl.py create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/processor_config.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/special_tokens_map.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/tokenizer_config.json create mode 100644 eval/pytorch_ref/policies/gr00t_n16/model/modules/nvidia/Eagle-Block2A-2B-v2/vocab.json create mode 100644 eval/pytorch_ref/policies/gr00t_n17/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/embodiment_configs.py create mode 100755 eval/pytorch_ref/policies/gr00t_n17/data/embodiment_tags.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/interfaces.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/state_action/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/state_action/action_chunking.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/state_action/pose.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/state_action/state_action_processor.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/types.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/data/utils.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/gr00t_n1d7/__init__.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/gr00t_n1d7/gr00t_n1d7.py create mode 100755 eval/pytorch_ref/policies/gr00t_n17/model/gr00t_n1d7/image_augmentations.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/gr00t_n1d7/processing_gr00t_n1d7.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/modules/__init__.py create mode 100755 eval/pytorch_ref/policies/gr00t_n17/model/modules/dit.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/modules/embodiment_conditioned_mlp.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/modules/flowmatching_modules.py create mode 100644 eval/pytorch_ref/policies/gr00t_n17/model/modules/qwen3_backbone.py create mode 100644 eval/pytorch_ref/policies/pi0.py create mode 100644 eval/pytorch_ref/policies/smolvla.py create mode 100644 eval/pytorch_ref/policies/torch_compile.py create mode 100644 eval/pytorch_ref/policies/vla_adapter/__init__.py create mode 100644 eval/pytorch_ref/policies/vla_adapter/model.py create mode 100644 eval/pytorch_ref/policies/vla_adapter/verify_ref.py create mode 100644 eval/pytorch_ref/pyproject.toml create mode 100644 eval/pytorch_ref/server/evo1_server.py create mode 100644 eval/pytorch_ref/server/gr00t_n15_server.py create mode 100644 eval/pytorch_ref/server/gr00t_n17_server.py create mode 100644 eval/pytorch_ref/server/gr00t_server.py create mode 100644 eval/pytorch_ref/server/pi0_server.py create mode 100644 eval/pytorch_ref/server/smolvla_server.py create mode 120000 eval/pytorch_ref/sim create mode 100644 eval/pytorch_ref/utils/service.py create mode 100644 eval/pytorch_ref/utils/sim_adapters/__init__.py create mode 100644 eval/pytorch_ref/utils/sim_adapters/base.py create mode 100644 eval/pytorch_ref/utils/sim_adapters/libero.py create mode 100644 eval/pytorch_ref/utils/sim_adapters/simpler.py create mode 100644 eval/pytorch_ref/uv.lock create mode 100755 eval/run_bitvla_compile_compare.sh create mode 100755 eval/run_bitvla_pytorch_perf.sh create mode 100755 eval/run_latency_compare.sh create mode 100755 eval/run_latency_compare_all.sh create mode 100755 eval/run_latency_serial.sh create mode 100755 eval/run_libero_bitvla.sh create mode 100755 eval/run_libero_compare.sh create mode 100755 eval/run_libero_pytorch.sh create mode 100755 eval/run_solver_step_displacement.sh create mode 100755 eval/run_solver_step_sweep.sh create mode 100755 eval/setup_bitvla_ref_env.sh diff --git a/ci/lib/common.sh b/ci/lib/common.sh index f2bff2c..7d0aa46 100755 --- a/ci/lib/common.sh +++ b/ci/lib/common.sh @@ -139,7 +139,7 @@ apply_gr00t_env() { local arch="$1" case "$arch" in gr00t_n1_5|gr00t_n1_6|gr00t_n1_7) - export VLA_GR00T_BF16_WEIGHTS="${VLA_GR00T_BF16_WEIGHTS:-1}" ;; + : ;; esac case "$arch" in gr00t_n1_5) export VLA_GR00T_EMBODIMENT="${VLA_GR00T_EMBODIMENT:-new_embodiment}" ;; diff --git a/docs/backend/sycl.md b/docs/backend/sycl.md index 9017f93..08dc291 100644 --- a/docs/backend/sycl.md +++ b/docs/backend/sycl.md @@ -217,6 +217,6 @@ weights) does not fit and dies in the allocator: level_zero backend failed with error: 38 (UR_RESULT_ERROR_OUT_OF_HOST_MEMORY) ``` -`VLA_GR00T_BF16_WEIGHTS=1` halves the weights but its activations still overflow +`--weight-dtype bf16` (now the default) halves the weights but its activations still overflow the card. There is no host-memory spill path - the core is single-backend - so the larger checkpoints need an A770/B580-class card or better. diff --git a/docs/corl-paper/experiments/compile_compare.md b/docs/corl-paper/experiments/compile_compare.md new file mode 100644 index 0000000..a05930f --- /dev/null +++ b/docs/corl-paper/experiments/compile_compare.md @@ -0,0 +1,406 @@ +# Inference latency — vla.cpp vs PyTorch (eager / `torch.compile`) + +- Generated: 2026-08-13 +- Branch: `fix/bf16-fusion` (`4d29e95`), llama.cpp pinned at `b10331` +- Suite: `libero_object`, task 0 +- Hardware: 1x RTX 3090 (24 GB), Intel i7-14700F (20C/28T) +- Harness: `eval/run_latency_serial.sh` -> `eval/run_latency_compare.sh` + +**This revision replaces an earlier one measured on branch `corl`. It is not a +re-run of the same experiment.** Three defects in that document's method were +found while reproducing it, and each changed a conclusion rather than a digit; +they are written up in [What the previous revision got wrong](#what-the-previous-revision-got-wrong) +because the corrections matter more than the new numbers. + +Server-side inference time per prediction. The vla.cpp server reports its own +latency per response; the PyTorch server times `select_action` in-process with +CUDA synchronisation on both sides. Neither includes ZeroMQ transport or image +serialisation. Both stacks run `n_action_steps = 1`, so every timed call is a +real forward pass rather than an action-queue pop. Warmup is excluded. + +**Every model was measured alone.** One model, one variant, one GPU, nothing +else on the machine. The previous revision used a two-lane driver that ran two +sweeps concurrently across both GPUs, which matters more than it sounds — see +[Why serial](#why-serial). + +## Results + +| Model | vla.cpp | PyTorch eager | compile (default) | compile (reduce-overhead) | best PyTorch | vla.cpp vs best | +|---|---:|---:|---:|---:|---:|---:| +| `smolvla` | **51.1**¹ | 164.4 | 64.0 | 59.3 | 59.3 | **1.16x faster** | +| `pi0` | **94.2**² | 102.0 | 102.0 | fails³ | 102.0 | **1.08x faster** | +| `evo1` | **137.5**² | 151.1 | 140.9 | 140.5 | 140.5 | **1.02x faster** | +| `gr00t_n1_5` | **68.7** | 94.3 | 88.6 | 87.9 | 87.9 | **1.28x faster** | +| `gr00t_n1_6` | **54.4** | 65.4 | 59.9 | fails³ | 59.9 | **1.10x faster** | +| `gr00t_n1_7` | **53.8** | 62.4 | 59.5 | fails³ | 59.5 | **1.11x faster** | +| `bitvla` | **52.1** | 319.7⁴ | 94.6⁴ | 94.0⁴ | 94.0 | **1.80x faster** | + +Milliseconds, lower is better. `vla.cpp vs best` is best-PyTorch / vla.cpp, so +>1x means vla.cpp wins. **vla.cpp is faster on 7/7.** + +¹ needs `VLA_MM_PREC=default`. This is not a tuning flag — it is the +configuration the shipped accuracy number was measured under. See +[smolvla and the precision that was never applied](#smolvla-and-the-precision-that-was-never-applied). +² needs `VLA_*_BF16_ACT=1` + `VLA_*_FA=1`. BF16 activations are bit-exact; +flash attention is not. See [Switches](#switches). +³ `reduce-overhead` (CUDA graphs) does not run on these: a tensor escapes the +compiled region and the graph memory pool overwrites it (`pi0`: a tensor in +lerobot's `sample_actions`; `gr00t_n1_6`: Eagle SigLIP2 caches `freqs_cis` as a +module attribute), and `gr00t_n1_7` aborts during capture. Fixing these means +editing the reference implementations, so "best PyTorch" is not drawn from an +equal menu across rows. +⁴ `bitvla`'s PyTorch side needs OpenVLA-OFT's `prismatic` and BitVLA's own +transformers fork, which cannot share the `pt_ref` venv, so it runs on +`eval/run_bitvla_compile_compare.sh`. Those three figures are carried over +unchanged from the previous revision; nothing in this branch touches the +PyTorch side. The vla.cpp figure is fresh. + +### Sample sizes + +PyTorch columns are n=300; vla.cpp is n=200. **Every vla.cpp figure comes from +one binary** (`4d29e95`) — the four rows this branch does not touch +(`gr00t_*`, `bitvla`) were re-measured on it to confirm, and moved by at most +0.6 ms. Spread makes the differing n immaterial: every vla.cpp row has p95 +within 2% of its median. + +### Distribution (median / p95, ms) + +| Model | vla.cpp | eager | compile (default) | compile (reduce-overhead) | +|---|---|---|---|---| +| `smolvla` | 51.0 / 51.9 | 164.0 / 166.4 | 63.4 / 67.4 | 59.0 / 61.1 | +| `pi0` | 94.1 / 95.2 | 102.0 / 103.7 | 102.0 / 103.4 | — | +| `evo1` | 137.6 / 138.4 | 150.8 / 153.8 | 140.4 / 143.8 | 139.8 / 143.4 | +| `gr00t_n1_5` | 68.6 / 69.3 | 82.8 / 138.9 | 80.6 / 129.2 | 79.7 / 124.6 | +| `gr00t_n1_6` | 54.4 / 54.8 | 65.3 / 66.8 | 59.8 / 60.8 | — | +| `gr00t_n1_7` | 53.8 / 54.2 | 62.1 / 64.0 | 58.4 / 59.4 | — | +| `bitvla` | 52.1 / 52.8 | — | — | — | + +vla.cpp's p95 sits within 2% of its median on every row. PyTorch's `gr00t_n1_5` +is the extreme case in the other direction: 82.8 median against 138.9 p95. For a +closed-loop controller the tail is usually what binds, so the mean-based table +understates vla.cpp's practical position — and on `gr00t_n1_5` it overstates +PyTorch's, because that mean is inflated by a tail rather than describing a +typical call. + +## What the previous revision got wrong + +### 1. `gr00t_n1_5`'s win was mostly PyTorch's host-side jitter + +The old revision called 1.20x on `gr00t_n1_5` a decisive kernel win. It is not a +kernel result. Instrumenting the reference's `select_action` in three phases +(`eval/pytorch_ref/policies/gr00t_n15/__init__.py`) over 300 calls: + +| phase | mean | med | p95 | min | max | +|---|---:|---:|---:|---:|---:| +| host preprocessing (lerobot pipeline) | 21.3 | 14.8 | **60.7** | 10.6 | **71.3** | +| model forward | 53.6 | 52.4 | 56.7 | 51.5 | 58.9 | +| D2H + unnormalize | 11.0 | 11.0 | 11.1 | 10.8 | 11.2 | + +The model is steady to +/-4 ms. All of the variance is host-side Python, +swinging 10.6 -> 71.3 ms per call, which `torch.compile` cannot touch — and +indeed `fwd` is 53.6 eager, 54.3 compiled, 54.5 with CUDA graphs. **Compile does +nothing to this model's forward.** The 100.5 -> 94.7 "compile win" the old +revision reported was preprocessing noise. + +On model time alone (`fwd + post` ~= 65 ms) PyTorch is *faster* than vla.cpp's +69.3. vla.cpp wins end-to-end because its preprocessing is ~5 ms of C++ against +lerobot's 10-71 ms of Python. That is a real deployment property and worth +having, but it is not the claim the old table made. + +`gr00t_n1_5`'s PyTorch numbers also move 6-7 ms run to run on an idle machine +with identical code, for the same reason. Do not read any single one of them +too precisely, including the ones in this table. + +### 2. Every profile was taken at the wrong granularity + +The old revision's kernel breakdown — *"only 53.6% of GPU time is in GEMM or +attention"*, and the three causes built on it — was measured with `nsys` +defaulting to `--cuda-graph-trace=graph`. That reports each CUDA **graph launch +as one entry** instead of itemising the kernels inside it. Every model here +replays CUDA graphs, so those profiles saw a small fraction of the real work: on +`evo1`, summing the reported kernels gives ~160 ms across 60 requests against +~8.8 s of actual GPU time — about 2%. + +**Treat that section of the old document as unusable, not merely imprecise.** +The corrected profile (`--cuda-graph-trace=node`) is what the `evo1` work below +is based on. Whether cause #2 (fusion, ~15%) and cause #3 (layout copies, 5.2%) +survive re-derivation is untested; they were sized from the bad profile. + +### 3. The baselines came from a different branch, not an earlier commit + +The old figures were measured on branch `corl`, whose `src/` differs from `main` +across 30 files: a pre-refactor `smolvla.cpp`, no `src/cuda/vla_cuda_bf16.cu` +(so BF16 support lived inside a multi-file ggml patch), and llama.cpp pinned at +**b9866** rather than b10331. So "post-merge regression" comparisons against it +were really `corl` vs `main`. + +Rebuilding `corl` reproduces its numbers on today's machine (`smolvla` 55.7 vs +55.8 published; 67.7 vs 68.5), so the old figures were sound *for that tree*. +They were just never a baseline for this one. + +## smolvla and the precision that was never applied + +`smolvla` measured 8.3 ms slower on `main` than on `corl`, which looked like a +regression and was reported as one. It is not. + +`smolvla` stamps `GGML_PREC_F32` on **every** tower weight matmul, via `mm_w()` +— unlike every other arch here, which sets it only on the `kq` attention-score +product where F32 accumulation buys softmax stability. llama.cpp b9866 silently +ignored that request for BF16 matmuls; b10331 honours it, routing all those +GEMMs onto the F32 cuBLAS path. + +The control is decisive — dropping the request changes nothing on the old build +and everything on the new one: + +| `smolvla`, FA on | F32 prec requested | BF16 prec (`VLA_MM_PREC=default`) | cost of honouring | +|---|---:|---:|---:| +| llama b9866 | 55.3 | 55.7 | **0.0 — ignored** | +| llama b10331 | 63.2 | **51.1** | 12.1 | + +So `main` is not slower; it is doing arithmetic we asked for and never previously +received. Two consequences: + +- The published 55.8 ms **and its 96/100 success rate** were both obtained at + BF16 GEMM precision. The old document describes a configuration that build + could not deliver. +- Today's default is therefore the *slower and unvalidated* one, and + `VLA_MM_PREC=default` is the *faster and already-measured* one. Flipping the + default returns to the validated configuration rather than departing from it. + +**Not yet flipped.** It is still a numerics change, and the argument rests on +inferring what an old build did rather than on a direct measurement. One +100-episode SR arm at `VLA_MM_PREC=default` on this build (~2 h, expect ~96/100) +would settle it. Until then the table's 51.1 requires the flag. + +## `evo1`: what moving kernels in-house cost, and getting it back + +Commit `5d9ec9e` (pre-dating this branch) moved BF16 activation support out of +ggml's own `binbcast.cu`/`norm.cu` and behind a single extension hook, with the +kernels reimplemented in `src/cuda/vla_cuda_bf16.cu`. That had three effects, all +discovered here: + +| state | `evo1` BF16+FA | +|---|---:| +| on merged `main`, before this branch | **crash** | +| fusion declined (`GGML_CUDA_DISABLE_FUSION`) | 169.5 | +| fused-add hook (`ad9a56b`) | 166.6 | +| row-addressed kernels (`0daf584`) | 146.9 | +| 8-wide vectorized rows (`4d29e95`) | **137.5** | +| `corl` at b9866, via ggml's own BF16 kernels | 142.3 | + +**The crash.** ggml fuses runs of elementwise nodes in +`ggml_backend_cuda_graph_compute`, *upstream* of `ggml_cuda_compute_forward` — +so the hook never got first refusal on a fused node, and +`ggml_cuda_op_fused_binbcast_impl` handles F32/F16 only and `GGML_ABORT`s on +BF16. `ad9a56b` adds one more exported pointer beside `ggml_cuda_ext_forward` +and one guarded call site in the fusion branch; the fused BF16 kernel lives in +`src/cuda/`. The llama.cpp patch remains three hunks. + +**The slowness, which was the larger problem.** Restoring fusion bought only +~3 ms. The real cost was the replacement kernels: `k_bin_bcast_bf16` recovered +`(i0,i1,i2,i3)` from a flat index with a 64-bit division and three modulos *per +element*, where ggml derives them from a 3D grid. `0daf584` addresses rows via +`blockIdx.y/z` (−19.7 ms); `4d29e95` then widens the contiguous case to 8 BF16 +per thread via one `uint4` (−9.4 ms). Both are **bit-identical** to the kernel +they replace — max |diff| 0.0 over 20 steps against `VLA_BF16_FLAT=1`, actions +compared with pinned flow-matching noise (`eval/compare_act_dtype.py`). + +`pi0` shares these kernels and moved 104.2 -> 96.7 -> **94.2** on the same two +commits. + +**The lesson is about ownership, not kernels.** Whatever ggml owns improves +without us; whatever we take in-house we maintain against upstream's rate. We +spent this work re-deriving indexing and vectorization that `binbcast.cu` +already had. + +## Bumping llama.cpp is not free in one direction + +Same vla.cpp source, only `VLA_LLAMA_TAG` differing (`b9866` was `corl`'s pin): + +| model | b9866 | b10331 | change | +|---|---:|---:|---:| +| `gr00t_n1_6` | 57.0 | 54.1 | −2.9 (−5.1%) | +| `gr00t_n1_5` | 71.7 | 69.3 | −2.4 (−3.3%) | +| `gr00t_n1_7` | 55.5 | 53.8 | −1.7 (−3.1%) | +| `evo1` | 148.6 | 146.9 | −1.7 (−1.1%) | +| `pi0` | 97.1 | 96.7 | −0.4 (−0.4%) | +| `bitvla` | 51.8 | 51.8 | **0.0** | +| `smolvla` | 55.3 | 63.8 | +8.5 (precision, above) | + +Five models got faster for nothing. **`bitvla`'s zero is the control**: its hot +path is our own `bitvla_cuda_kernels`, not ggml's, so it is insulated from ggml +kernel work — and it is the one row that did not move. That is evidence the +gains are ggml's kernels rather than anything environmental. + +Which upstream commits did it is unknown; b9866 -> b10331 is ~460 tags and only +`smolvla` was profiled across them. The policy conclusion is not "pin forward", +it is **bump regularly and re-measure every model**, because direction is not +uniform and one row moved 8 ms against us while five moved 2-3 ms for us. + +`VLA_LLAMA_TAG` (CMake cache) and `SERVER_BIN` (harness) exist so two builds can +be compared without editing tracked files. + +## Recommended flags + +### Build — one configuration for every model + +There is no per-model build flag. vla.cpp exposes exactly two CMake options of +its own (`VLA_LLAMA_TAG`, `VLA_BUILD_TESTS`); everything performance-relevant +comes from ggml and is already the default. + +```sh +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 # 86 = RTX 3090 (sm_86) +cmake --build build -j"$(nproc)" +``` + +Defaults worth knowing are already on and should stay on: `GGML_CUDA_GRAPHS=ON` +(capture/replay — every model here replays, see +[the audit](#what-the-previous-revision-got-wrong)), `GGML_CUDA_FA=ON`, +`GGML_CUDA_FORCE_MMQ=OFF`, `GGML_CUDA_FORCE_CUBLAS=OFF`. Set +`CMAKE_CUDA_ARCHITECTURES` to your own SM version; a mismatch costs a JIT compile +on first launch. + +Leave `VLA_LLAMA_TAG` at its default `b10331`. It exists to bisect upstream, not +to tune — and pinning back is the wrong move even though it would win 8 ms on +`smolvla`, because it loses 2-3 ms each on five other models +([why](#bumping-llamacpp-is-not-free-in-one-direction)). + +**Rebuilding across `5d9ec9e` with a warm `build/` fails**, because +`build/_deps/llama-src` still carries the older multi-file ggml patch and the +current single-hook patch cannot find its anchor. The script is idempotent for an +already-hooked tree but not for a differently-patched one. Fix: + +```sh +git -C build/_deps/llama-src checkout HEAD -- ggml/src/ggml-cuda/ggml-cuda.cu +``` + +### Run — per model + +Fastest measured configuration for each. `VLA_POLICY_DIR`/`HF_HOME` are omitted; +set them as your deployment needs. + +| Model | run flags | ms | bit-exact vs default? | +|---|---|---:|---| +| `smolvla` | `VLA_SMOLVLA_FA=1 VLA_MM_PREC=default` | 51.1 | no — both change numerics | +| `pi0` | `VLA_PI0_BF16_ACT=1 VLA_PI0_FA=1` | 94.2 | BF16 yes, FA no | +| `evo1` | `VLA_EVO1_BF16_ACT=1 VLA_EVO1_FA=1` | 137.5 | BF16 yes, FA no | +| `gr00t_n1_5` | `VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=new_embodiment` | 68.7 | yes | +| `gr00t_n1_6` | `VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=libero_panda` | 54.4 | yes | +| `gr00t_n1_7` | `VLA_GR00T_BF16_WEIGHTS=1` | 53.8 | yes | +| `bitvla` | *(none)* | 52.1 | yes | + +`VLA_GR00T_EMBODIMENT` is a correctness flag, not a performance one — it selects +the normalization statistics, and the wrong value produces wrong actions rather +than slower ones. + +### If you want speed without changing numerics + +Every flash-attention flag and `smolvla`'s `VLA_MM_PREC=default` alter results; +the rest do not. Dropping the non-exact ones: + +| Model | bit-exact flags | ms | cost vs fastest | +|---|---|---:|---| +| `smolvla` | *(none)* | 74.6 | +23.5 | +| `pi0` | `VLA_PI0_BF16_ACT=1` | 96.7 | +2.5 | +| `evo1` | `VLA_EVO1_BF16_ACT=1` | ~147 | +9.5 | +| `gr00t_n1_5/6/7`, `bitvla` | as above — already exact | — | 0 | + +At bit-exact settings vla.cpp still wins six of seven; only `smolvla` flips +(74.6 against 59.3). Note the asymmetry in what "exact" is worth: it costs +`smolvla` 31% and `pi0` under 3%, because `pi0`'s 256-token tower barely +benefits from flash attention in the first place. + +`evo1`'s bit-exact figure is interpolated from its pre-vectorization measurement +rather than measured on `4d29e95`; the vectorized kernels are bit-exact, so only +the FA component is being removed. + +### Debug and bisect flags + +Not for production; they exist to make regressions findable. + +| Flag | effect | +|---|---| +| `VLA_BF16_FLAT=1` | scalar BF16 elementwise path instead of vectorized; bit-identical, so an A/B isolates indexing bugs from arithmetic ones | +| `VLA_BITVLA_NARROW_GEMM=1` | pre-retiling ternary GEMM | +| `GGML_CUDA_DISABLE_FUSION=1` | stock ggml switch; declines elementwise fusion | +| `VLA_MM_PREC=default` | listed above, but also the control for the precision finding | +| `SERVER_BIN=` | harness override to measure a second build | +| `VLA_LLAMA_TAG=` | CMake cache, to build against another llama.cpp | + +All boolean flags read their **value** since `1efb7a3`; before that, `=0` turned +them on. + +## Switches + +| Model | switches for the table figure | bit-exact? | +|---|---|---| +| `smolvla` | `VLA_SMOLVLA_FA=1`, `VLA_MM_PREC=default` | FA no; MM_PREC no (but see above) | +| `pi0` | `VLA_PI0_BF16_ACT=1`, `VLA_PI0_FA=1` | BF16 yes, FA no | +| `evo1` | `VLA_EVO1_BF16_ACT=1`, `VLA_EVO1_FA=1` | BF16 yes, FA no | +| `gr00t_n1_5/6/7` | `VLA_GR00T_BF16_WEIGHTS=1` | yes | +| `bitvla` | none | yes | + +`VLA_GR00T_BF16_WEIGHTS` is **not** a shipping default: `gr00tn1d*.cpp` selects +F32 when it is unset, and `eval/run_latency_compare.sh` exports `=1` for the +GR00T models. So launching `vla-server` by hand without it measures a different +precision than the table. What the F32 default costs is unmeasured. + +**All boolean switches now read their value.** Until `1efb7a3` they tested only +presence, so `VLA_EVO1_FA=0` *enabled* flash attention. That silently invalidates +any A/B done by setting a switch to zero, and it cost one run in this session +before being found. 20 switches across 12 files were converted; `0`, `false`, +`off`, `no` and empty are now false. + +### Flash attention is the one unsettled trade + +ggml's CUDA flash attention computes K/V at **F16 regardless of input type** +(`fattn.cu:247` reinterprets an F32 K/V tensor as F16), so there is no +full-precision FA path on this backend. Passing F32 K/V produced byte-identical +actions, which is how this was established. + +The gain tracks tokens² because score-matrix traffic dominates: ~1024-token +towers gain 15-25%, `pi0`'s 256-token tower ~3%. Against that, two models showed +a ~4-5 pp SR drop in the same direction (`evo1` 97->92, `smolvla` 96->92, +`pi0` 88->89 i.e. none) — but the `evo1` half **did not reproduce** in a later +same-binary 2x2 (96 with, 96 without), and at n=100 with p~0.95 a 4 pp effect is +~1.3 SE, inconclusive either way. + +So FA ships **opt-in** not because it is known-bad but because a measured 15-25% +is being traded against an unmeasured accuracy risk, and for a robot policy that +is the wrong direction to leave uncertainty. Settling it needs ~1,450 episodes +per arm (~7 h) on `evo1` and `smolvla`, which is where the gain lives; `pi0`'s +3% does not justify any risk. + +One untested opportunity: `gr00t_n1_7`'s vision tower has its own +`VLA_FLASH_ATTN` switch (`src/models/qwen3vl_vit.h`), also default off, so its +53.8 ms was measured without FA. Its 64 merged tokens suggest little to gain. + +## Why serial + +The previous revision's driver dealt models across both GPUs and ran two lanes +concurrently, so every figure was measured with a second full sweep — another +server, another LIBERO sim — competing for the same 20 cores. That is not a +detail: the PyTorch rows spend 10-70 ms per call in host-side Python +(`gr00t_n1_5`, above) which absorbs CPU contention directly. + +It also produced a mismatch inside one row: `gr00t_n1_5`'s published vla.cpp +figure came from the contended two-lane run while its PyTorch figures came from +a later solo re-run — a comparison that ran *against* vla.cpp. `eval/run_latency_serial.sh` +runs one model, one variant, one GPU at a time. + +## Open + +1. **SR arm at `VLA_MM_PREC=default`** for `smolvla` (~2 h). Gates flipping the + default, worth 19% on that row. +2. **Powered SR for flash attention** on `evo1` and `smolvla`, ~1,450 eps/arm. + The weakest evidence in this document. +3. **Re-derive causes #2 and #3** with `--cuda-graph-trace=node`. The old + sizings came from graph-granularity profiles and cannot be trusted. +4. **`evo1`'s remaining elementwise cost.** A corrected profile still puts our + BF16 kernels at ~17 ms/call of 137.5 (`k_bin_bcast` 2,276 launches/call for + the largest). Much of that is launch count, not per-launch efficiency, so the + next lever is fewer nodes or more fusion rather than faster kernels. +5. **`bitvla` builds five graphs per `predict()`** with no caching, the only arch + without it. Whether that costs anything is unmeasured — `smolvla` turned out + to be replaying CUDA graphs fine, so this is less promising than it looks. diff --git a/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-mac.md b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-mac.md new file mode 100644 index 0000000..5843160 --- /dev/null +++ b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-mac.md @@ -0,0 +1,317 @@ +# SmolVLA on the UR10e through vla.cpp - Apple M4 / Metal + +One session, 10 rollouts, 2026-08-08 07:39:56 - 07:54:21. Same checkpoint, same +GGUF, same client and same bridge as the CUDA and CPU sessions in +`UR-SMOLVLA-VLA-CPP.md`; the engine is `vla-server` on the Mac Mini M4's GPU. + +| | | +|---|---| +| Engine | `vla-server`, Metal backend, `VLA_WEIGHT_DTYPE=f32` | +| Box | Mac Mini M4, 10 cores, 24 GB unified, `192.168.56.91` | +| Data | `rollouts_m4_vlacpp1/` | +| Labels | operator, 10/10 | +| Topology | client (container, this host) -> bridge + engine (Mac), over the wired segment | + +Unlike sessions G and C, the server is **not** on the client's box: every +`metadata.json` says `server: 192.168.56.91:8791`, so the LAN is in the loop. +Setup and the Metal parity result are in RUN_SMOLVLA.md, +"On the Mac Mini (Metal)". Every number below comes from the `states.npz` / +`metadata.json` files and the recorded video. + +**Ten episodes, not twenty.** Every success-rate statement here rests on n = 10, +which is half the sample of the sessions it is compared against. The latency +numbers come from 293 queries and are solid; the success rate is not. + +## Summary + +**Metal makes the Mac Mini twice the policy server vla.simd made it.** 582 ms per +query against 1177 ms for vla.simd on the same box, and 402 ms of that is the +engine. Dwell between chunks halves, the control rate goes from 9.7 Hz to +12.8 Hz, and the arm spends 30% of the session waiting instead of 46%. + +**A quarter of the round trip is cable, not compute.** 160.1 ms of the 582.0 ms is +the network, and it is the most predictable number in the session: p95 161.7 ms, +max 162.9 ms over 293 queries. That is two raw 640x480 frames, 1.8 MB, on a +**100BASE-TX** segment. On gigabit the same query lands at ~440 ms. The engine is +now small enough that the wire is the second-biggest term in the loop. + +**Success rate: 6/10 (60%),** the same as vla.simd on this box, the same as +vla.cpp on the host CPU, and the same as the Ryzen. Against the CUDA session's +75%, Fisher p = 0.43. Nothing here separates Metal from any other way of running +this policy. + +**The failure is the documented release failure, unchanged.** All 10 episodes +grasp the cup and carry it to the basket. The 4 failures then hold it just outside +the rim and oscillate there until the operator stops the run, idle 16.9 - 21.4 s. +Grasp and transport did not fail once. + +**Latency still does not predict success.** This session is 5.2x slower than the +CUDA one and scores within noise of it; steps to first grasp is 374 +- 44 against +CUDA's 335 +- 26. Four backends spanning 111 ms to 2343 ms per query have now all +landed between 60% and 75%. + +## Headline comparison + +Sessions G and C are from `UR-SMOLVLA-VLA-CPP.md` (same host, engine on the +client's box). Session A is `rollouts_m4/`, vla.simd on **this same Mac**, from +`REPORT_SMOLVLA.md`. + +| Metric | M: vla.cpp Metal | G: vla.cpp CUDA | C: vla.cpp CPU | A: vla.simd M4 | +|---|---|---|---|---| +| Episodes | 10 | 20 | 20 | 20 | +| Success | 6/10 (60%) | 15/20 (75%) | 12/20 (60%) | 12/20 (60%) | +| Server round trip, mean | 582.0 ms | 111.4 ms | 2164.4 ms | 1176.6 ms | +| Engine inference, mean | 402.3 ms | 88.3 ms | 2142.0 ms | n/a | +| Bridge preprocessing | 19.6 ms | 21.1 ms | 20.6 ms | n/a | +| Network | **160.1 ms** | 2.0 ms (loopback) | 1.8 ms (loopback) | in the 1177 | +| Round trip p95 / p99 / max | 634 / 638 / 651 ms | 126 / 148 / 152 ms | 2301 / 2424 / 2492 ms | | +| Round-trip spread (episode means) | 575.7 - 597.8 ms (3.8%) | 109.4 - 114.8 ms (4.9%) | 2131 - 2211 ms (3.7%) | 6% | +| Coefficient of variation | 0.055 | 0.069 | 0.038 | | +| Wall clock blocked on the server | 30.0% | 8.0% | 62.1% | 46.1% | +| Effective control rate | 12.76 Hz | 17.68 Hz | 7.11 Hz | 9.74 Hz | +| Chunk cycle, p50 | 1800 ms | 1300 ms | 3400 ms | | +| Steps to first grasp | 374 +- 44 | 335 +- 26 | 329 +- 41 | 384 +- 37 | +| Steps per successful episode | 782 (1.08x a demo) | 835 (1.16x) | 857 (1.19x) | 951 (1.32x) | +| Session wall time | 9.5 min | 14.8 min | 35.2 min | 28.3 min | +| Steps / queries recorded | 7,244 / 293 | 15,681 / 637 | 15,030 / 606 | | +| Dropped recording ticks | 0 | 0 | 0 | | +| Slew-limited commands | 0 | 0.08% | 0.04% | | +| Stop reason | `interrupted` x10 | `interrupted` x20 | `interrupted` x20 | | + +## Latency + +### Where the round trip goes + +Three measurement points: `latency_ms` by the client (request sent -> chunk +received), `handling_ms` by the bridge (the whole request), `inference_ms` by the +engine. The differences give the network and the bridge's own preprocessing. + +| Stage | Mean | What it is | +|---|---|---| +| Engine (`vla-server`, Metal) | 402.3 ms | vision tower + prefill + denoise on the M4 GPU | +| Bridge (`handling - inference`) | 19.6 ms | resize-with-pad to 512x512 x2, tokenizer lookup, protobuf, ZMQ on loopback | +| Network (`latency - handling`) | 160.1 ms | 1.8 MB of raw frames out, the pickled chunk back, on 100BASE-TX | +| **Client round trip** | **582.0 ms** | | + +### The network term + +| | Value | +|---|---| +| Mean | 160.1 ms | +| p95 / max | 161.7 / 162.9 ms | +| Spread over 293 queries | ~3 ms | + +This is a wire, not a queue: 1.8 MB of uncompressed frames at 100 Mbit is ~150 ms +of serialisation, and nothing in the session perturbs it. Both ends report +100BASE-TX (`ifconfig en0 | grep media` on the Mac, +`cat /sys/class/net/enp2s0/speed` here). A gigabit switch and cable removes ~145 ms +of it, taking this configuration from 582 ms to ~440 ms and the control rate from +12.8 Hz to about 15 Hz - no code, and it helps the vla.simd path by exactly the +same amount. + +### Distribution + +| | Value | +|---|---| +| Round trip mean / p50 | 582.0 / 559.1 ms | +| p95 / p99 / max | 633.9 / 638.0 / 651.1 ms | +| min | 549.8 ms | +| Coefficient of variation | 0.055 | +| Engine mean / p50 / p95 / max | 402.3 / 388.4 / 439.9 / 462.8 ms | + +Stable across the session: episode-mean round trip stays inside +575.7 - 597.8 ms (3.8%) and episode-mean engine drifts 399.1 -> 414.7 ms, a ~4% +rise over 9.5 minutes that is consistent with the fanless M4 warming up. No +thermal cliff, no dropped ticks, no slew-limited commands and no server errors +across 293 queries. + +### What the client does with it + +At `--exec-horizon 25` the client executes 25 actions (1.25 s at 20 Hz) and then +blocks on the next chunk. Measured chunk cycle is 1800 ms p50: 1250 ms of motion +plus ~0.58 s of dwell. That is 30.0% of the session spent waiting, against 8.0% on +CUDA and 46.1% for vla.simd on this same Mac. + +The arm visibly pauses between chunks, at half the vla.simd pause. It is the best +this cell does without the workstation GPU. + +## Success rate and labels + +Operator labels, in order: + +``` +P F F P P F P P F P +``` + +6 passes (ep 1, 4, 5, 7, 8, 10), 4 failures (ep 2, 3, 6, 9). **6/10 = 60%**, +Wilson 95% CI 31 - 83%. + +| Session | Engine | RT | Success | Wilson 95% CI | +|---|---|---|---|---| +| M (this) | vla.cpp Metal, M4 | 582 ms | 6/10 (60%) | 31-83% | +| G | vla.cpp CUDA | 111 ms | 15/20 (75%) | 53-89% | +| C | vla.cpp CPU | 2164 ms | 12/20 (60%) | 39-78% | +| A | vla.simd, same M4 | 1177 ms | 12/20 (60%) | 39-78% | +| B | vla.simd, Ryzen 7 | 2343 ms | 12/20 (60%) | 39-78% | + +| Comparison | Fisher exact | +|---|---| +| M vs G (CUDA) | p = 0.43 | +| M vs C (host CPU) | p = 1.00 | +| M vs A (vla.simd on this same Mac) | p = 1.00 | + +Metal lands exactly on the 60% that three of the four earlier sessions produced. +The CI is wide - 31 to 83% on ten episodes - so this is evidence of *no +difference*, not evidence of equality; it would take a much longer session to +separate 60% from 75%. + +## Per-episode results + +`1st grasp` = step of the first gripper CLOSE. `Last grip` = time of the last +gripper command. `Idle` = seconds from that command to the operator's stop. +`Drift` = median over the episode of `max|action - measured joints|`. + +| Ep | Result | Rollout | Steps | Wall s | Q | RT ms | Engine ms | Net ms | 1st grasp | Last grip s | Idle s | Drift rad | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | pass | `073956` | 675 | 53.6 | 27 | 575.7 | 399.1 | 160.6 | 320 | 51.6 | 1.6 | 0.0058 | +| 2 | **fail** | `074209` | 590 | 46.0 | 24 | 580.3 | 401.7 | 160.3 | 372 | 29.1 | 16.9 | 0.0064 | +| 3 | **fail** | `074325` | 589 | 46.0 | 24 | 582.2 | 402.1 | 159.9 | 309 | 26.1 | 19.8 | 0.0069 | +| 4 | pass | `074452` | 752 | 59.4 | 31 | 580.6 | 400.8 | 160.2 | 411 | 57.0 | 2.3 | 0.0053 | +| 5 | pass | `074624` | 850 | 64.9 | 34 | 579.4 | 400.7 | 157.1 | 387 | 64.6 | 0.3 | 0.0048 | +| 6 | **fail** | `074813` | 725 | 59.8 | 29 | 580.8 | 400.2 | 160.6 | 449 | 39.5 | 20.0 | 0.0055 | +| 7 | pass | `074941` | 758 | 59.4 | 31 | 584.4 | 404.4 | 160.6 | 365 | 59.3 | 0.1 | 0.0063 | +| 8 | pass | `075115` | 872 | 68.2 | 35 | 577.8 | 398.5 | 160.5 | 420 | 64.2 | 3.9 | 0.0046 | +| 9 | **fail** | `075300` | 646 | 50.1 | 26 | 579.2 | 399.9 | 160.5 | 348 | 28.7 | 21.4 | 0.0060 | +| 10 | pass | `075421` | 787 | 60.3 | 32 | 597.8 | 414.7 | 160.8 | 355 | 59.3 | 0.5 | 0.0053 | + + +## The failure mode + +Contact sheet of every last frame, green = pass: +`figures_vlacpp/end_side_m4_metal.jpg`. Fail and pass side by side at full +resolution: `figures_vlacpp/end_m4_metal_ep09_fail_vs_ep10_pass.jpg`. + +**Grasp and transport never failed.** All 10 episodes closed the gripper on the +cup - first close at step 374 +- 44, about 19 s of policy time - and carried it to +the basket. Nothing failed before the release, which is what every earlier session +on this robot found. + +**All 4 failures are the same release stall.** The arm arrives holding the cup, +hovers at or just outside the rim, and oscillates there until the operator stops +the run. The telemetry signature: + +| Signal | Failures (4) | Successes (6) | +|---|---|---| +| Idle after the last gripper command | 16.9 - 21.4 s | 0.1 - 3.9 s | +| Gripper state at the end | closed, all 4 | open, or closing back within ~1 s of a release | +| Steps per episode | 638 +- 64 | 782 +- 72 | + +The detector from `REPORT_SMOLVLA.md` - gripper closed at the end **and** idle +> 10 s - fires on all 4 failures with 0 false alarms on the 6 successes. The idle +gap is the cleanest separator in the data: 16.9 s at the low end of the failures +against 3.9 s at the high end of the successes, with nothing in between. + +**Drift does not separate them this time.** Successes span 0.0046 - 0.0063 rad and +failures 0.0055 - 0.0069, and ep7 (pass, 0.0063) sits above ep6 (fail, 0.0055). +`REPORT_SMOLVLA.md` already flagged drift as a signal that looks like a perfect +classifier on one session and stops being one on the next; this session is another +instance. Use the idle gap. + +## Latency and policy behaviour + +Two things are worth reading off the comparison table. + +**Steps to first grasp does not track latency.** 374 +- 44 here at 582 ms per +query, against 335 +- 26 at 111 ms and 329 +- 41 at 2164 ms. The 2.3x spread in +approach length across sessions has no relationship to the 19x spread in server +latency. + +**Successful episodes are the shortest recorded on this robot** - 782 steps, 1.08x +the 721-step average demo, against 1.16x on CUDA, 1.19x on the host CPU and 1.32x +for vla.simd on this same Mac. On ten episodes that is 6 successes' worth of +evidence and could easily be scene luck, so it is an observation, not a claim. + +What the latency does buy is the dwell: 0.58 s between chunks instead of vla.simd's +1.18 s on the same hardware. At `--exec-horizon 25` that is 0.58 s of pause per +1.25 s of motion - visible, but far from the vla.simd stop-and-go. + +## Infrastructure + +- The Mac is offline - one interface on the robot segment, no default route - so + the engine was built there from a source tree, a pinned llama.cpp and a set of + wheels staged over rsync by `scripts/deploy_vlacpp_mac.sh`. Details in + RUN_SMOLVLA.md, "On the Mac Mini (Metal)". +- `vla-server` ran the Metal backend with `VLA_WEIGHT_DTYPE=f32`: 1447.5 MiB of + unified memory for the towers, against 775.6 MiB at the bf16 default. f32 is + required on this backend - bf16 fails the parity check at 2.0e-03 rad on the + joints where f32 gives 8.3e-05. +- `metadata.json` does not record the dtype, but the timings confirm it: the + minimum engine time over 293 queries is 379.9 ms and p05 is 382.2 ms, against an + idle-box benchmark of 363 ms for bf16 and 388 - 396 ms for f32. The session never + approached the bf16 floor. +- The bridge ran with `--tokens 18188,614,260,7118,198`, which pins the + instruction and keeps transformers off the Mac. Every `metadata.json` handshake + confirms `task: "pick up the cup"`. +- The client was unchanged: `--server 192.168.56.91:8791`, `--exec-horizon 25`. +- Image age at request time is p50 17.1 ms, the same as every other session: the + 160 ms of network happens *after* the frame is grabbed, so it does not make the + observation the policy sees any staler. + +## Caveats + +- **n = 10.** Wilson 95% CI on 6/10 is 31 - 83%. This session can tell you Metal is + not obviously broken; it cannot rank Metal against CUDA. +- **Labels are the operator's**, applied live during the run. +- **The engine dtype is inferred, not recorded.** The timing argument is strong but + indirect. Worth passing the engine's backend and dtype through the bridge + handshake into `metadata.json`. +- **Session A (vla.simd on this Mac) is from 2026-07-29**, so the same-box + comparison spans a scene reset and ten days. +- The network term is specific to this cabling. Re-measure after any switch or + cable change: `latency_ms - handling_ms` in any rollout gives it directly. + +## Recommendations + +1. **Put the segment on gigabit.** 145 ms of the 582 ms round trip is + serialisation on a 100 Mbit link. It costs a cable and a switch port, needs no + code, and helps the vla.simd path identically. +2. **Keep `VLA_WEIGHT_DTYPE=f32` on Metal.** It costs 33 ms and 672 MiB and buys + 24x on joint parity; the bf16 default fails the check. See RUN_SMOLVLA.md. +3. **Run 20 episodes next time**, and interleave them with a CUDA block in the + same sitting if the goal is to compare backends rather than to check one works. +4. **Record the backend and dtype in `metadata.json`.** The bridge already knows + both from the engine banner. +5. **The release is still the whole problem.** Four backends spanning 111 ms to + 2343 ms per query, and every failure in every session is the release. Grasp and + transport are solved; no inference hardware has moved the number, and none + will. + +## Reproducing these numbers + +```bash +python3 - <<'EOF' +import json, numpy as np +from pathlib import Path +LAB = "P F F P P F P P F P".split() +lat=[];hd=[];inf=[];steps=0;wall=0.0 +for d,l in zip(sorted(Path("rollouts_m4_vlacpp1").glob("rollout_*")), LAB): + m=json.loads((d/"metadata.json").read_text()); z=np.load(d/"states.npz") + lat.append(z["query_latency_ms"]); hd.append(z["query_handling_ms"]) + inf.append(z["query_inference_ms"]) + steps+=m["steps"]; wall+=m["duration_s"] + fl=np.where(np.diff(z["sent_close"].astype(int))!=0)[0] + idle=z["t"][-1]-z["t"][fl[-1]] + print(d.name[-6:], l, m["steps"], "rt %.0f" % m["timings_ms"]["server_roundtrip"]["mean"], + "grasp@%d" % z["step"][fl[0]], "idle %.0fs" % idle, + "STALL" if (z["state_grip"][-1] < 0.5 and idle > 10) else "") +lat,hd,inf = map(np.concatenate,(lat,hd,inf)) +print(f"n={len(lat)} rt {lat.mean():.1f} (p95 {np.percentile(lat,95):.1f}) " + f"engine {inf.mean():.1f} bridge {(hd-inf).mean():.1f} net {(lat-hd).mean():.1f} " + f"blocked {100*lat.sum()/1000/wall:.1f}% rate {steps/wall:.2f} Hz " + f"SR {LAB.count('P')}/{len(LAB)}") +EOF +``` + +Figures in `figures_vlacpp/` (the rollout directories are owned by another user +and not writable): `end_side_m4_metal.jpg` (2x5 contact sheet of every last frame, +green border = pass) and `end_m4_metal_ep09_fail_vs_ep10_pass.jpg`. diff --git a/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-ryzen7.md b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-ryzen7.md new file mode 100644 index 0000000..7de252a --- /dev/null +++ b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla-ryzen7.md @@ -0,0 +1,467 @@ +# SmolVLA on the UR10e through vla.cpp - Ryzen 7 ultrabook / CPU + +One session, 10 rollouts, 2026-08-08 09:17:39 - 09:47:36. Same checkpoint, same +GGUF, same client and same bridge as every other vla.cpp session; the engine is +`vla-server` on the ultrabook's CPU. + +| | | +|---|---| +| Engine | `vla-server`, CPU backend, `VLA_SMOLVLA_FA=1 VLA_WEIGHT_DTYPE=f32` | +| Box | Ryzen 7 PRO 6850HS ultrabook, 8 cores / 16 threads, 15 GB, `192.168.56.11` | +| Data | `rollouts_ryzen7_vlacpp/` | +| Labels | operator, 10/10 | +| Topology | client (container, this host) -> bridge + engine (ultrabook), over the wired segment | + +**This is the same physical box as session B**, the vla.simd Ryzen session in +`REPORT_SMOLVLA.md` (`rollouts_ryzen7_vlasimd/`, 2026-07-29). Only its IP changed, +`192.168.56.71` -> `192.168.56.11`. That makes this the first pair in the whole +series that swaps the *engine* with the hardware held fixed. Setup and the parity +result are in RUN_SMOLVLA.md, "On the ultrabook (CPU)". + +**Ten episodes, not twenty.** Every success-rate statement rests on n = 10. The +latency numbers come from 297 queries and are solid; the success rate is not. + +## Summary + +**vla.cpp is 1.56x slower than vla.simd on this box, and that is the expected +trade.** 3653 ms per query against 2343 ms, 4.98 Hz against 6.73 Hz. vla.cpp buys +one runtime and one GGUF across CPU, CUDA and Metal at roughly twice the CPU +latency; this session prices that trade on identical hardware instead of across +boxes. + +**The package power limit is visible in the robot data, not just on the bench.** +The first query of every episode averages 2551 ms and the rest average 3493 ms - +the first query is **27% faster**, in ten out of ten episodes. This is the +opposite sign from CUDA, where the first query is 38% *slower* (graph warm-up). +Here the box has been idle between episodes, so it starts each one with a full +boost budget and spends it in the first query. Nothing warms up; something runs +out. + +**The robot's duty cycle is worth 19%.** Back-to-back on the bench this engine +sustains 4247 ms. In the session, with 1.25 s of arm motion between queries, it +averages 3461 ms. The idle gap partially refills the boost budget every cycle. A +benchmark that hammers the server understates the robot number by a fifth, and one +that fires a single query overstates it by a third. + +**Success rate: 7/10 (70%),** the highest of the three CPU sessions, and not +distinguishable from any of them: Fisher p = 0.70 against both 12/20 sessions, +p = 1.00 against CUDA's 15/20. Wilson 95% CI is 40 - 89%. Do not read a quality +gap into it. + +**The failure is the documented release failure, unchanged.** All 10 episodes +grasp the cup and carry it to the basket. The 3 failures then hold it outside the +rim and oscillate until the operator stops the run, idle 46 - 56 s against 0.1 - +0.5 s on the passes. Grasp and transport did not fail once. + +**The "latency costs steps" law is now dead on its own hardware.** On this box, +vla.simd at 2343 ms took 459 +- 44 steps to first grasp; vla.cpp at 3653 ms takes +**348 +- 26** (Welch t = -8.7, df = 27). The engine got 1.56x slower and the +policy reached the grasp in 111 fewer steps. Whatever produced session B's slow +grasp, it was not the latency. See +[Latency and policy behaviour](#latency-and-policy-behaviour). + +## Headline comparison + +Session B is `rollouts_ryzen7_vlasimd/` on **this same box**. G and C are from +`UR-SMOLVLA-VLA-CPP.md` (engine on the client's box), M from +`UR-SMOLVLA-VLA-CPP-MAC.md`. + +| Metric | R: vla.cpp Ryzen CPU | B: vla.simd, same box | C: vla.cpp host CPU | M: vla.cpp Metal | G: vla.cpp CUDA | +|---|---|---|---|---|---| +| Episodes | 10 | 20 | 20 | 10 | 20 | +| Success | 7/10 (70%) | 12/20 (60%) | 12/20 (60%) | 6/10 (60%) | 15/20 (75%) | +| Server round trip, mean | **3652.9 ms** | 2342.6 ms | 2164.4 ms | 582.0 ms | 111.4 ms | +| Engine inference, mean | 3461.2 ms | n/a | 2142.0 ms | 402.3 ms | 88.3 ms | +| Bridge preprocessing | 29.7 ms | n/a | 20.6 ms | 19.6 ms | 21.1 ms | +| Network | **162.0 ms** | in the 2343 | 1.8 ms (loopback) | 160.1 ms | 2.0 ms (loopback) | +| Round trip p95 / p99 / max | 3817 / 4034 / 4578 ms | | 2301 / 2424 / 2492 ms | 634 / 638 / 651 ms | 126 / 148 / 152 ms | +| Round-trip spread (episode means) | 3605 - 3738 ms (3.6%) | | 2131 - 2211 ms (3.7%) | 576 - 598 ms (3.8%) | 109 - 115 ms (4.9%) | +| Coefficient of variation | 0.056 | | 0.038 | 0.055 | 0.069 | +| Wall clock blocked on the server | **73.2%** | 63.3% | 62.1% | 30.0% | 8.0% | +| Effective control rate | **4.98 Hz** | 6.73 Hz | 7.11 Hz | 12.76 Hz | 17.68 Hz | +| Chunk cycle, p50 | 4900 ms (98 ticks) | | 3400 ms (68 ticks) | 1800 ms (36 ticks) | 1300 ms (26 ticks) | +| Steps to first grasp | **348 +- 26** | 459 +- 44 | 329 +- 41 | 374 +- 44 | 335 +- 26 | +| Steps per successful episode | 786 (1.09x a demo) | | 857 (1.19x) | 782 (1.08x) | 835 (1.16x) | +| Session wall time | 24.7 min | 46.3 min | 35.2 min | 9.5 min | 14.8 min | +| Steps / queries recorded | 7,379 / 297 | 18,726 / 751 | 15,030 / 606 | 7,244 / 293 | 15,681 / 637 | +| Dropped recording ticks | 0 | | 0 | 0 | 0 | +| Slew-limited commands | 0% | | 0.04% | 0% | 0.08% | +| Stop reason | `interrupted` x10 | | `interrupted` x20 | `interrupted` x10 | `interrupted` x20 | + +## Latency + +### Where the round trip goes + +Three measurement points: `latency_ms` by the client (request sent -> chunk +received), `handling_ms` by the bridge (the whole request), `inference_ms` by the +engine. The differences give the network and the bridge's own preprocessing. + +| Stage | Mean | Share | What it is | +|---|---|---|---| +| Engine (`vla-server`, CPU f32) | **3461.2 ms** | 94.7% | SigLIP tower + prefill + 10 flow-matching steps | +| Bridge (`handling - inference`) | 29.7 ms | 0.8% | resize-with-pad to 512x512 x2, tokenizer lookup, protobuf, ZMQ on loopback | +| Network (`latency - handling`) | 162.0 ms | 4.4% | 1.8 MB of raw frames out, the pickled chunk back, on 100BASE-TX | +| **Client round trip** | **3652.9 ms** | | | + +The engine is 95% of the query here, against 79% on CUDA. Everything outside it is +noise at this speed: the same 162 ms of cable that costs the Mac 27% of its query +costs this box 4%. + +### The network term + +162.0 ms mean, p95 162.6 ms, max 178.2 ms over 297 queries - the same wire, and +the same stability, the Mac session measured (160.1 ms). Both ends of this segment +report 100BASE-TX. A gigabit switch would remove ~145 ms of it, which here is 4% +of the query and not worth doing for this path alone. It is worth doing for the +Metal path, where the same 145 ms is a quarter of the query. + +### Distribution + +| | Round trip | Engine | +|---|---|---| +| min | 2696.3 | 2508.4 | +| p50 | 3683.5 | 3492.2 | +| p95 | 3817.1 | 3623.1 | +| p99 | 4033.5 | | +| max | 4578.0 | 4380.5 | +| mean | 3652.9 | 3461.2 | +| sd | 205.9 | | +| CV | 0.056 | | + +All in ms, 297 queries. The mins are the per-episode first queries, not outliers - +see below. + +### The power limit, measured on the robot + +**The first query of each episode is the fastest one in it, every time:** + +| | Engine | Round trip | +|---|---|---| +| First query of an episode (n = 10) | **2551 ms** | 2746 ms | +| Every other query (n = 287) | **3493 ms** | 3684 ms | +| Difference | **-27%** | -25% | + +The per-episode firsts span 2508 - 2606 ms, a 4% spread across ten episodes. This +is not noise and it is not a cache: it is the 28 W package starting each episode +with an unspent boost budget after ~20 s of operator idle between runs. + +Within an episode it decays over the first three or four queries and then flattens: + +| Query # in episode | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | +|---|---|---|---|---|---|---|---|---| +| Engine, mean ms | 2551 | 3349 | 3423 | 3459 | 3460 | 3525 | 3622 | 3478 | + +**This is the opposite sign from CUDA.** There the first query of an episode is +38% *slower* (120.5 ms against 87.3 ms) because the graph and kernels re-warm +after an idle gap. Here idle makes the box *faster*. Two different machines, two +opposite first-query artifacts, and on this one it is worth 0.9 s. + +**No session-level drift.** Episode-mean engine time runs 3414 - 3540 ms with no +trend (ep 1 is the highest at 3540, ep 10 is 3437), and the package sits at +64 - 68 C under load against a ~95 C Tjmax. The limit is power, not heat, and it +is reached within one episode rather than over the session. + +### Bench against robot + +The bench numbers in RUN_SMOLVLA.md were taken back to back, each query issued the +moment the last chunk arrived. The robot leaves 1.25 s of arm motion between them: + +| Duty cycle | Engine | Round trip | +|---|---|---| +| Back to back (bench, 34 queries) | 4247 ms | 4444 ms | +| Robot, `--exec-horizon 25` (297 queries) | **3461 ms** | **3653 ms** | +| Single query into an idle box | 2551 ms | 2746 ms | + +A 1.7x spread on the same binary, the same GGUF and the same box, decided entirely +by how much idle time sits between queries. **Quote the duty-cycle-matched +number.** Neither the hammered bench nor the single shot describes the robot. + +### What the client does with it + +The client ticks at 20 Hz, executes `exec_horizon = 25` actions from each chunk, +then issues the next query and blocks on it: + +| | Measured | +|---|---| +| Execute 25 actions | 1250 ms | +| Blocked on the query | 3653 ms | +| Chunk cycle, p50 | **4900 ms** = 98 ticks | +| Chunk cycle, mean | 4939 ms | +| Dwell fraction | 75% of the cycle | +| Predicted rate | 5.10 Hz | +| Measured session rate | **4.98 Hz** | + +Three ticks of standing still for every one of motion. This is the slowest of the +five backends and it looks it: the arm moves for 1.25 s and then holds for 3.7 s. + +### Observation staleness + +The frame the policy actually sees is ~17 ms old (`img_age` p50), the same as +every other session - the cameras run at ~30 Hz and the sampling is unaffected by +the backend. What differs is how stale it is by the time the actions derived from +it reach the arm: + +| | This session | CUDA | +|---|---|---| +| Age at the 1st action of the chunk | 3670 ms | 128 ms | +| Age at the 25th action | 4920 ms | 1378 ms | + +The whole CUDA chunk is acted on ~2.7x fresher than this session's *freshest* +action. + +## Success rate and labels + +Operator labels, in order: + +``` +P F P P P P P F P F +``` + +7 passes (ep 1, 3, 4, 5, 6, 7, 9), 3 failures (ep 2, 8, 10). **7/10 = 70%,** +Wilson 95% CI 40 - 89%. + +| Session | Engine | RT | Success | Wilson 95% CI | +|---|---|---|---|---| +| R (this) | vla.cpp CPU, Ryzen 7 | 3653 ms | 7/10 (70%) | 40-89% | +| B | vla.simd, **same box** | 2343 ms | 12/20 (60%) | 39-78% | +| C | vla.cpp CPU, host | 2164 ms | 12/20 (60%) | 39-78% | +| M | vla.cpp Metal, M4 | 582 ms | 6/10 (60%) | 31-83% | +| G | vla.cpp CUDA | 111 ms | 15/20 (75%) | 53-89% | +| A | vla.simd, M4 | 1177 ms | 12/20 (60%) | 39-78% | + +| Comparison | Fisher exact | +|---|---| +| R vs B (vla.simd, same box) | p = 0.70 | +| R vs C (host CPU) | p = 0.70 | +| R vs G (CUDA) | p = 1.00 | +| R vs M (Metal) | p = 1.00 | + +70% is the best number any CPU session has produced and it means nothing on its +own: with 10 episodes the interval runs from 40% to 89%, and it overlaps every +other session in the series. Six sessions spanning 111 ms to 3653 ms per query +have now all landed between 60% and 75%. + +## Per-episode results + +`1st grasp` = step of the first gripper CLOSE. `Last grip` = time of the last +gripper command. `Idle` = seconds from that command to the operator's stop. +`Drift` = median over the episode of `max|action - measured joints|`. + +| Ep | Result | Rollout | Steps | Wall s | Q | RT ms | Engine ms | Net ms | 1st grasp | Last grip s | Idle s | Drift rad | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | pass | `091739` | 800 | 162.9 | 32 | 3738.0 | 3540.4 | 162.5 | 384 | 160.0 | 0.1 | 0.0045 | +| 2 | **fail** | `092042` | 600 | 119.7 | 24 | 3650.1 | 3455.3 | 161.9 | 362 | 72.0 | 46.2 | 0.0063 | +| 3 | pass | `092315` | 700 | 141.1 | 28 | 3641.1 | 3450.8 | 161.8 | 307 | 138.1 | 0.4 | 0.0063 | +| 4 | pass | `092618` | 775 | 155.0 | 31 | 3617.3 | 3426.6 | 161.9 | 331 | 150.8 | 0.5 | 0.0053 | +| 5 | pass | `092921` | 832 | 167.8 | 34 | 3604.9 | 3414.4 | 161.9 | 321 | 167.5 | 0.2 | 0.0046 | +| 6 | pass | `093240` | 775 | 155.8 | 31 | 3688.0 | 3497.3 | 161.9 | 339 | 153.0 | 0.4 | 0.0047 | +| 7 | pass | `093553` | 761 | 151.7 | 31 | 3641.9 | 3451.5 | 162.0 | 364 | 151.0 | 0.4 | 0.0053 | +| 8 | **fail** | `093853` | 650 | 130.0 | 26 | 3668.9 | 3478.1 | 161.9 | 384 | 82.4 | 46.9 | 0.0061 | +| 9 | pass | `094135` | 861 | 173.2 | 35 | 3648.9 | 3458.3 | 162.0 | 343 | 172.5 | 0.5 | 0.0047 | +| 10 | **fail** | `094530` | 625 | 125.7 | 25 | 3627.9 | 3437.1 | 162.1 | 340 | 66.9 | 55.6 | 0.0066 | + +Drift is 0.0045 - 0.0066 rad, the same band as every other session, so the +controller is tracking the policy exactly as well here as it does at 17.7 Hz. + +## The failure mode + +Unchanged: transport succeeds, release does not. In all 3 failures the last +gripper command is a CLOSE issued long before the operator stopped, the measured +gripper reads 0.000, and the last frame shows the cup still clamped next to the +basket. + +**The stall detector agrees with the operator on 10 of 10.** Rule: *measured +gripper closed at the end AND no gripper command for more than 10 s*. Its margin +here is the widest in the series - failures idle 46.2, 46.9 and 55.6 s while +passes idle 0.1 - 0.5 s, so the 10 s threshold has two orders of magnitude of +headroom on both sides. Across six sessions and 89 episodes it now stands at +**32/32 failures caught, 0 false alarms**. + +**Joint travel separates cleanly again.** Over the last 20 s: + +| | Failures | Successes | +|---|---|---| +| This session | 0.64 - 0.77 rad (mean 0.70) | 0.21 - 0.40 rad (mean 0.29) | +| Host CPU (C) | 1.05 - 1.22 rad (mean 1.14) | 0.27 - 0.52 rad (mean 0.35) | +| CUDA (G) | 1.81 - 2.36 rad (mean 2.10) | 0.83 - 2.04 rad (mean 1.51) | + +The separation holds on both slow backends and collapses on CUDA, which is the +rate artifact `UR-SMOLVLA-VLA-CPP.md` called out: at 17.7 Hz a *successful* +episode also covers a lot of ground in its last 20 s, because it is still +executing 350 actions in that window. At 5 Hz it is executing 100. Travel is not a +rate-portable signal; the gripper-state detector is. + +**End pose against the 44 training demonstrations.** Release pose = last frame of +a success, stuck pose = last frame of a failure. Training statistics quoted from +`UR-SMOLVLA-VLA-CPP.md`: + +| Joint | Train release | sd | Release / z | Stuck / z | +|---|---|---|---|---| +| shoulder_pan | -1.0391 | 0.019 | -1.0624 / -1.2 | -1.0760 / -1.9 | +| shoulder_lift | -1.9178 | 0.026 | -1.8654 / +2.0 | -1.8460 / **+2.8** | +| elbow | -1.5323 | 0.031 | -1.5816 / -1.6 | -1.5300 / +0.1 | +| wrist_1 | -1.2655 | 0.025 | -1.2689 / -0.1 | -1.3405 / **-3.0** | +| wrist_2 | +1.5713 | 0.0001 | +1.5712 | +1.5713 | +| wrist_3 | -2.6082 | 0.019 | -2.6336 / -1.3 | -2.6438 / -1.9 | + +RMS z over the five live joints: **release 1.41, stuck 2.19**. Successful releases +land inside the demonstrations' spread; the stuck pose sits outside it, driven by +`wrist_1` at z = -3.0. The two poses agree to 0.072 rad, so this is one cluster +sampled two ways, not a second failure mode. `wrist_2` is dead, as `DATASET.md` +documents. + +## Latency and policy behaviour + +This session settles a question the earlier reports left open. + +`REPORT_SMOLVLA.md` compared the M4 (1.18 s per query, 384 +- 37 steps to grasp) +against this same Ryzen box running vla.simd (2.34 s, 459 +- 44) and read the ++19% as closed-loop degradation from latency. `UR-SMOLVLA-VLA-CPP.md` then failed +to reproduce it across CUDA and the host CPU and warned the figure should not be +carried forward. This session tests it **on the box that produced it**: + +| Session | Box | Engine | Round trip | Steps to first grasp | +|---|---|---|---|---| +| B | Ryzen 7 6850HS | vla.simd | 2343 ms | 459 +- 44 | +| R (this) | **same box** | vla.cpp CPU | **3653 ms** | **348 +- 26** | +| | | | | Welch t = -8.7, df = 27 | + +**The engine got 1.56x slower and the policy reached the grasp 111 steps +earlier.** With the hardware, the cell, the client and the checkpoint all held +fixed, latency moved the wrong way for the hypothesis and the effect is large and +significant. The +19% law does not survive. + +All six sessions, ordered by round trip: + +| Session | Server | Round trip | Steps to first grasp | Rate | +|---|---|---|---|---| +| `rollouts_vlacpp_gpu` | vla.cpp CUDA | 111 ms | 335 +- 26 | 17.68 Hz | +| `rollouts_m4_vlacpp1` | vla.cpp Metal, M4 | 582 ms | 374 +- 44 | 12.76 Hz | +| `rollouts_m4_vlasimd` | vla.simd, M4 | 1177 ms | 384 +- 37 | 9.74 Hz | +| `rollouts_vlacpp_cpu1` | vla.cpp CPU, host | 2164 ms | 329 +- 41 | 7.11 Hz | +| `rollouts_ryzen7_vlasimd` | vla.simd, Ryzen | 2343 ms | 459 +- 44 | 6.73 Hz | +| `rollouts_ryzen7_vlacpp` | vla.cpp CPU, Ryzen | **3653 ms** | **348 +- 26** | 4.98 Hz | + +Steps-to-grasp spans 329 to 459 with no relationship to latency, and the slowest +session in the series sits in the middle of the range. Against this session, +CUDA (t = +1.3) and the host CPU (t = +1.5) are indistinguishable across a 33x +latency ratio; only the two vla.simd sessions are slower to grasp. + +What that leaves is an engine-shaped residual: the two highest steps-to-grasp +numbers are the two vla.simd sessions, and the four vla.cpp sessions sit at +329 - 374 regardless of backend or speed. That is consistent with the SigLIP +position-id gotcha `VLACPP.md` documents, which the vla.simd path hit by a +different mechanism and which shifts actions by ~2e-02 rad while still looking +plausible. **This session does not establish that** - the two runs are 10 days +apart with no scene measurement between them - but it is the hypothesis worth +testing next, and it is cheap to test: re-run session B's conversion through the +current parity check. + +Steps per successful episode tell the same story: 786 here (1.09x a 721-step +demonstration), the second-tightest in the series behind Metal's 782, against +857 on the host CPU and 835 on CUDA. + +## Infrastructure + +| Metric | Value | +|---|---| +| Episodes / steps / queries | 10 / 7,379 / 297 | +| Recorded wall time | 24.7 min | +| Blocked on the server | 1085 s of 1483 s = 73.2% | +| Effective control rate | 4.98 Hz | +| Dropped recording ticks | 0 | +| Step period, non-query ticks | p50 50.00 ms, p95 50.21 ms | +| Ticks with step period > 100 ms | 4.08% (one per chunk) | +| Slew-limited commands | 0 | +| Server errors | 0 | +| Stop reason | `interrupted` x10 | + +Zero errors across 297 queries. The engine and the bridge ran for 30 minutes of +continuous serving with no restart, on a laptop, over the wired segment. The 20 Hz +timer itself is exact - 50.00 ms at p50 on ticks that do not carry a query - so +the entire rate loss is the blocking query. + +## Caveats + +- **n = 10, one scene, one operator, one day.** 70% has a Wilson interval of + 40 - 89% and is consistent with every other session in the series. Nothing here + ranks backends by success. +- **The comparison against session B is 10 days apart.** Same box and same + checkpoint, but the cup start pose was not measured on either day, and + `REPORT_SMOLVLA.md` already flags B's labels as inferred rather than the + operator's. The steps-to-grasp gap is large enough (t = -8.7) that scene drift + is an unlikely full explanation, but it is not excluded. +- **Stop times are the operator's.** Episodes end on Ctrl+C, so step counts and + durations partly measure operator patience. "Failures are shorter" is close to + circular; the stall-then-abort structure is the content. +- **The power-limit numbers depend on the duty cycle,** which depends on + `--exec-horizon` and on how long the operator waits between episodes. At + `--exec-horizon 50` the gaps are longer and the engine should sit somewhere + between the 3461 ms measured here and the 2551 ms single-shot figure. That was + not measured. +- **The bench numbers in the duty-cycle table are mine, not the session's,** taken + the same day on the same binary and GGUF through the same bridge, but with a + synthetic client on real frames rather than the robot. +- **Numerics are not compared here.** The parity result for this box is in + RUN_SMOLVLA.md: 5.8e-05 rad against torch fp32 with the f32 towers this session + ran, which is tighter than any other spot in the series. +- Nothing here measures grasp force or whether the cup was held securely. + +## Recommendations + +1. **Use `--exec-horizon 50` on this box.** At 25 the arm dwells 75% of the + session; at 50 the same query cost is amortised over 2.5 s of motion, which + takes the rate from 5.0 Hz to ~7.2 Hz for free. This is the one configuration + change that matters here. +2. **Keep vla.simd as the fast CPU path on this box.** 2343 ms against 3653 ms on + identical hardware. Reach for vla.cpp here when the single-runtime, single-GGUF + property is worth 1.56x, which is a real reason - it is the same binary and the + same file that run on CUDA and Metal. +3. **Quote duty-cycle-matched latency for power-limited boxes.** The same engine + measures 2551, 3461 or 4247 ms depending only on the gap between queries. Bench + this class of machine the way the robot will drive it, or the number is + fiction. +4. **Re-test session B's conversion.** The four vla.cpp sessions cluster at + 329 - 374 steps to grasp and the two vla.simd sessions sit at 384 and 459. Run + the current parity check against the vla.simd Ryzen setup before attributing + any of that to the policy. +5. **Ship the stall detector.** 10/10 here, 32/32 failures with 0 false alarms + over 89 episodes and five backends. Count the idle in policy steps rather than + seconds so the threshold survives across a 3.5x range of control rates. +6. **The release is still the bug.** Three failures, all at the basket, all with + grasp and transport intact - now demonstrated at 4.98 Hz as well as at 7.1 and + 17.7 Hz. The fix is demonstrations that release from a range of poses around + the basket, the gap `DATASET.md` calls out, not a faster server. + +## Reproducing these numbers + +```bash +python3 - <<'EOF' +import json, numpy as np +from pathlib import Path +root, lab = "rollouts_ryzen7_vlacpp", list("PFPPPPPFPF") +lat, inf, hd, steps, wall, first = [], [], [], 0, 0.0, [] +for d, l in zip(sorted(Path(root).glob("rollout_*")), lab): + m = json.loads((d/"metadata.json").read_text()) + z = np.load(d/"states.npz") + lat.append(z["query_latency_ms"]); inf.append(z["query_inference_ms"]) + hd.append(z["query_handling_ms"]); first.append(z["query_inference_ms"][0]) + steps += m["steps"]; wall += m["duration_s"] + sc, t, g = z["sent_close"], z["t"], z["state_grip"] + fl = np.where(np.diff(sc.astype(int)) != 0)[0] + idle = t[-1] - t[fl[-1]] + print(root, d.name[8:], l, m["steps"], + "rt %.0fms" % m["timings_ms"]["server_roundtrip"]["mean"], + "eng %.0fms" % m["timings_ms"]["server_inference"]["mean"], + "grasp@%d" % z["step"][fl[0]], "idle %.0fs" % idle, + "STALLED" if (g[-1] < 0.5 and idle > 10) else "ok") +rest = np.concatenate([a[1:] for a in inf]) +lat, inf, hd = map(np.concatenate, (lat, inf, hd)) +print(f"{root}: rt {lat.mean():.1f} (p95 {np.percentile(lat,95):.1f}) " + f"engine {inf.mean():.1f} bridge {(hd-inf).mean():.1f} " + f"net {(lat-hd).mean():.1f} blocked {100*lat.sum()/1000/wall:.1f}% " + f"rate {steps/wall:.2f} Hz") +print(f"first query of an episode {np.mean(first):.0f} ms vs rest {rest.mean():.0f} ms " + f"({100*(np.mean(first)-rest.mean())/rest.mean():+.0f}%)") +EOF +``` diff --git a/docs/corl-paper/experiments/ur10e-vlacpp-smolvla.md b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla.md new file mode 100644 index 0000000..3cc64de --- /dev/null +++ b/docs/corl-paper/experiments/ur10e-vlacpp-smolvla.md @@ -0,0 +1,441 @@ +# SmolVLA on the UR10e through vla.cpp - CUDA vs CPU + +Two sessions, 40 rollouts, 2026-08-07. Same checkpoint, same GGUF, same client, +same bridge, same host. The only difference is which `vla-server` binary answered: + +| Session | Engine | Data | Labels | +|---|---|---|---| +| G | `build/ReleaseCUDA/vla-server` | `rollouts_vlacpp_gpu/` | operator, 20/20 | +| C | `build/ReleaseCPU/vla-server`, `VLA_SMOLVLA_FA=1 VLA_WEIGHT_DTYPE=f32` | `rollouts_vlacpp_cpu1/` | operator 19/20, ep20 from video | + +Host: i9-14900HX + RTX 5070 Laptop (Blackwell, 8 GB). Client, bridge and engine +all on this box - `server: 127.0.0.1:8791` in every `metadata.json`, so unlike the +earlier M4 / Ryzen sessions there is no network in the loop. Every number below +comes from the `states.npz` / `metadata.json` files and the recorded video. + +## Summary + +**Latency is the whole story, and it is a 19x gap.** 111 ms per query on CUDA +against 2164 ms on CPU. The engine is 24x apart (88 ms vs 2142 ms); everything +around it - the bridge preprocessing and the loopback socket - is identical to +within 1 ms. Both are metronome-stable across a full session: 3.7 - 4.9% spread +in episode means, no thermal drift, zero dropped ticks, zero server errors over +1243 queries. + +**What that buys on the robot is the control rate: 17.7 Hz against 7.1 Hz.** On +CUDA the client spends 8% of the session blocked on the server; on CPU it spends +62%. At `--exec-horizon 25` a chunk cycle is 26 ticks on CUDA (1.30 s, of which +0.11 s is dwell) and 68 ticks on CPU (3.40 s, of which 2.16 s is dwell). CUDA is +the first configuration on this robot that gets close to the 20 Hz the policy was +trained at. + +**Success rate: 15/20 (75%) on CUDA, 12/20 (60%) on CPU.** The difference is not +significant (Fisher p = 0.50; Wilson 95% CIs 53-89% and 39-78%, heavily +overlapping). Do not read a quality gap into it. + +**The failure is the same one documented in `REPORT_SMOLVLA.md`, unchanged.** All +13 failures across both sessions are a failed *release*: the policy grasps the cup +and carries it to the basket in 40 of 40 episodes, then in the failures holds it +just outside the rim and oscillates there until the operator stops the run. Grasp +and transport never failed on either backend. + +**One earlier conclusion does not replicate.** `REPORT_SMOLVLA.md` reported that +doubling latency (1.18 s -> 2.34 s) cost 19% more steps to reach the grasp. Here +latency goes up 19x and steps-to-grasp does not move: 335 +- 26 on CUDA against +329 +- 41 on CPU (Welch t = 0.57). See +[Latency and policy behaviour](#latency-and-policy-behaviour). + +## Headline comparison + +| Metric | G: vla.cpp CUDA | C: vla.cpp CPU | Ratio | +|---|---|---|---| +| Success | 15/20 (75%) | 12/20 (60%) | Fisher p = 0.50 | +| Server round trip, mean | **111.4 ms** | **2164.4 ms** | 19.4x | +| Engine inference, mean | 88.3 ms | 2142.0 ms | 24.3x | +| Round trip p95 / p99 / max | 126 / 148 / 152 ms | 2301 / 2424 / 2492 ms | | +| Round-trip spread (episode means) | 109.4 - 114.8 ms (4.9%) | 2131 - 2211 ms (3.7%) | | +| Coefficient of variation | 0.069 | 0.038 | | +| Wall clock blocked on the server | **8.0%** | **62.1%** | | +| Effective control rate | **17.68 Hz** | **7.11 Hz** (nominal 20 Hz) | 2.49x | +| Chunk cycle, p50 | 1300 ms (26 ticks) | 3400 ms (68 ticks) | | +| Steps to first grasp | 335 +- 26 | 329 +- 41 | t = 0.57, n.s. | +| Time to first grasp | 17.6 +- 1.3 s | 45.5 +- 5.5 s | 2.6x | +| Steps per successful episode | 835 (1.16x a demo) | 857 (1.19x a demo) | | +| Session wall time | 14.8 min | 35.2 min | 2.4x | +| Steps / queries recorded | 15,681 / 637 | 15,030 / 606 | | +| Dropped recording ticks | 0 | 0 | | +| Slew-limited commands | 0.08% | 0.04% | | +| Stop reason | `interrupted` x20 | `interrupted` x20 | | + +## Latency + +### Where the round trip goes + +Each query is measured at three points: `latency_ms` by the client (request sent -> +chunk received), `handling_ms` by the bridge (`vlacpp_policy_server.py`, the whole +request), `inference_ms` by the engine (`vla-server`, `latency_ms_total`). +Subtracting gives the loopback socket and the bridge's own preprocessing. + +| Stage | CUDA mean | CPU mean | What it is | +|---|---|---|---| +| Engine inference | **88.3 ms** | **2142.0 ms** | SigLIP tower + prefill + 10 flow-matching steps, inside `vla-server` | +| Bridge preprocessing | 21.1 ms | 20.6 ms | resize-with-pad 480x640 -> 512x512 x2, tokenizer lookup, protobuf + ZMQ hop | +| Pickle loopback | 2.0 ms | 1.8 ms | client <-> bridge over 127.0.0.1, 1.84 MB of raw frames | +| **Round trip** | **111.4 ms** | **2164.4 ms** | | +| Inference share of round trip | 79.3% | 99.0% | | + +The bridge costs the same 21 ms either way - it is numpy and a dict lookup on the +host CPU, unaffected by the backend. On CUDA that is a fifth of the whole query +and the largest remaining target; on CPU it is 1% and irrelevant. + +### Distribution + +| Percentile | CUDA round trip | CUDA engine | CPU round trip | CPU engine | +|---|---|---|---|---| +| min | 102.7 | 82.0 | 1934.4 | 1911.7 | +| p50 | 109.5 | 86.7 | 2151.1 | 2128.9 | +| p90 | 116.8 | 90.7 | 2270.4 | 2246.7 | +| p95 | 126.0 | 102.4 | 2300.7 | 2276.4 | +| p99 | 147.8 | 123.7 | 2423.9 | 2401.1 | +| max | 152.0 | 130.5 | 2491.9 | 2466.5 | +| sd | 7.7 | 6.9 | 82.5 | 82.1 | + +All in ms, 637 and 606 queries. The CPU path is *relatively* tighter (CV 0.038 vs +0.069) but its absolute jitter is 11x larger: a p99 outlier costs 36 ms on CUDA +and 260 ms on CPU, which is 5 extra ticks of dwell. + +**CUDA has a warm-up query, CPU does not.** The first query of each episode +averages 120.5 ms against 87.3 ms for the rest (+38%); the CPU first query is +2134 ms against 2142 ms, i.e. nothing. That is the CUDA graph / kernel path +re-warming after the bridge has been idle between episodes. It costs 33 ms once +per episode and is not worth chasing. + +**Neither backend degraded over a session.** CUDA episode means: first five 88.6 +ms, last five 88.3 ms. CPU: first five 2135 ms, last five 2155 ms. Full-session +spread 2.7 ms (3.1%) and 79.7 ms (3.7%). No thermal ramp on either, over 15 and +35 minutes of continuous load on a laptop. + +### What the client does with it + +The client ticks at 20 Hz, executes `exec_horizon = 25` actions from each chunk, +then issues the next query and **blocks** on it. Ticks are quantized to 50 ms, so +the cycle rounds up: + +| | CUDA | CPU | +|---|---|---| +| Execute 25 actions | 1250 ms | 1250 ms | +| Blocked on the query | 111 ms | 2164 ms | +| Chunk cycle, p50 | **1300 ms** = 26 ticks | **3400 ms** = 68 ticks | +| Chunk cycle, mean | 1383 ms | 3464 ms | +| Dwell fraction | 4% of the cycle | 64% of the cycle | +| Predicted rate | 18.4 Hz | 7.3 Hz | +| Measured session rate | 17.68 Hz | 7.11 Hz | + +On CUDA the pause between chunks is one tick. The arm looks continuous. On CPU it +is 43 ticks of standing still per 25 of motion. + +### Observation staleness + +`img_age` is recorded after the tick's work, so on the tick that carries a query +it reads the frame age *plus* the query duration. Netting that out, the frame +actually sent to the policy is ~17 ms old on both paths - the cameras run at +~30 Hz and both backends sample equally fresh. + +What differs is how old that observation is by the time each action derived from it +reaches the arm: + +| | CUDA | CPU | +|---|---|---| +| Age at the 1st action of the chunk | 128 ms | 2181 ms | +| Age at the 25th action of the chunk | 1378 ms | 3431 ms | +| Worst observed | 185 ms | 2516 ms | + +The whole chunk on CUDA is acted on fresher than the *first* action on CPU. + +### The `VLA_SMOLVLA_FA=1` gain, on the robot + +`rollouts_vlacpp_cpu/` is an earlier CPU session on the same host without flash +attention in the vision tower. Same client, same GGUF: + +| CPU session | Engine mean | Engine p50 | Round trip mean | Episode-mean spread | Rate | +|---|---|---|---|---|---| +| `rollouts_vlacpp_cpu` (no FA) | 2350 ms | 2371 ms | 2374 ms | 16.4% | 6.69 Hz | +| `rollouts_vlacpp_cpu1` (FA) | **2142 ms** | 2129 ms | **2164 ms** | 3.7% | **7.11 Hz** | + +-8.8% on the engine, +6% control rate, and the run-to-run spread collapses from +16.4% to 3.7%. This confirms the bench number in `VLACPP.md` (~2.3 s -> ~2.0 s) on +the robot. `VLA_SMOLVLA_FA=1` should stay in the documented CPU command. + +## Success rate and labels + +| | CUDA | CPU | +|---|---|---| +| Operator labels | 20 of 20 | 19 of 20 | +| Pass | 15 | 11 labeled + ep20 | +| Success rate | 15/20 = 75% | 12/20 = 60% | +| Wilson 95% CI | 53 - 89% | 39 - 78% | + +**On the label count.** The supplied CPU list has 19 entries against 20 rollouts. +Aligning it to the first 19 rollouts in timestamp order is self-consistent: the +stall detector (below) fires on exactly the 8 episodes marked `F` and on none of +the 11 marked `P`, with no misalignment possible. That leaves +`rollout_20260807_102344` unlabeled. Its last frame shows the blue cup sitting in +the basket with the gripper open above it, and its signals match the pass group +exactly (idle 0.1 s, no stall) - so it is scored as a pass, and both numbers are +given above. `figures_vlacpp/end_cpu_ep18_fail_vs_ep20_pass.jpg` is that frame +next to a confirmed failure. + +**The stall detector from `REPORT_SMOLVLA.md` holds on both sessions.** Rule: +*measured gripper closed at the end AND no gripper command for more than 10 s*. It +agrees with the operator on **39 of 39** labeled episodes here - 13/13 failures, +0 false alarms on 26 successes - and it now stands at 29/29 failures with 0 false +alarms over 79 episodes and four backends. + +Margin note: on CUDA the pass/fail idle gap is 6.4 s vs 13.3 s, against 2.5 s vs +28.7 s on CPU. Episodes are 2.5x shorter in wall clock, so the 10 s threshold has +less headroom. It still separates cleanly, but if the control rate rises further, +count the idle in policy steps rather than seconds. + +## Per-episode results + +`1st grasp` = step of the first gripper CLOSE. `Last grip` = time of the last +gripper command. `Idle` = seconds from that command to the operator's stop. +`Drift` = median over the episode of `max|action - measured joints|`. + +### Session G - vla.cpp CUDA (`rollouts_vlacpp_gpu/`) + +| Ep | Result | Rollout | Steps | Wall s | Q | RT ms | Engine ms | 1st grasp | Last grip s | Idle s | Drift rad | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | pass | `085124` | 756 | 44.4 | 31 | 111.8 | 87.8 | 320 | 43.6 | 0.2 | 0.0055 | +| 2 | pass | `085238` | 761 | 44.5 | 31 | 113.2 | 88.8 | 318 | 43.8 | 0.3 | 0.0059 | +| 3 | pass | `085358` | 766 | 44.7 | 31 | 113.3 | 88.9 | 311 | 42.4 | 1.8 | 0.0060 | +| 4 | **fail** | `085501` | 639 | 34.4 | 26 | 114.8 | 89.8 | 338 | 17.8 | 16.6 | 0.0066 | +| 5 | pass | `085601` | 940 | 55.3 | 38 | 112.1 | 87.7 | 346 | 53.6 | 1.7 | 0.0044 | +| 6 | **fail** | `085712` | 575 | 33.1 | 23 | 111.5 | 89.1 | 333 | 17.8 | 15.3 | 0.0070 | +| 7 | pass | `085816` | 828 | 47.4 | 34 | 110.0 | 87.5 | 340 | 45.6 | 1.9 | 0.0046 | +| 8 | pass | `085919` | 735 | 41.5 | 30 | 109.8 | 87.5 | 338 | 35.1 | 6.4 | 0.0057 | +| 9 | pass | `090022` | 835 | 47.8 | 34 | 109.4 | 87.4 | 319 | 46.3 | 1.5 | 0.0045 | +| 10 | pass | `090128` | 1053 | 57.2 | 43 | 111.5 | 88.4 | 339 | 56.0 | 0.0 | 0.0037 | +| 11 | pass | `090242` | 811 | 46.7 | 33 | 111.5 | 88.8 | 416 | 45.1 | 1.6 | 0.0049 | +| 12 | **fail** | `090346` | 545 | 29.5 | 22 | 112.0 | 89.5 | 308 | 16.2 | 13.3 | 0.0081 | +| 13 | pass | `090435` | 917 | 50.9 | 37 | 111.5 | 88.7 | 363 | 48.1 | 1.8 | 0.0043 | +| 14 | pass | `090548` | 869 | 48.5 | 35 | 110.3 | 87.8 | 312 | 45.5 | 2.9 | 0.0051 | +| 15 | pass | `090653` | 747 | 42.1 | 30 | 110.5 | 88.3 | 326 | 39.0 | 3.1 | 0.0057 | +| 16 | pass | `090803` | 764 | 44.2 | 31 | 111.8 | 88.7 | 316 | 42.3 | 1.9 | 0.0055 | +| 17 | pass | `090903` | 791 | 45.7 | 32 | 112.6 | 89.5 | 315 | 45.4 | 0.2 | 0.0055 | +| 18 | **fail** | `091021` | 733 | 40.3 | 30 | 110.8 | 88.4 | 313 | 17.4 | 23.0 | 0.0060 | +| 19 | pass | `091201` | 953 | 52.9 | 39 | 109.4 | 87.0 | 363 | 52.8 | 0.0 | 0.0046 | +| 20 | **fail** | `091311` | 663 | 35.6 | 27 | 111.0 | 88.2 | 362 | 18.9 | 16.6 | 0.0066 | + +### Session C - vla.cpp CPU (`rollouts_vlacpp_cpu1/`) + +| Ep | Result | Rollout | Steps | Wall s | Q | RT ms | Engine ms | 1st grasp | Last grip s | Idle s | Drift rad | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | pass | `094404` | 750 | 105.2 | 30 | 2194.6 | 2171.7 | 257 | 103.1 | 1.7 | 0.0052 | +| 2 | **fail** | `094602` | 575 | 79.1 | 23 | 2130.9 | 2109.0 | 361 | 49.5 | 29.0 | 0.0064 | +| 3 | **fail** | `094741` | 525 | 76.2 | 21 | 2136.9 | 2114.5 | 268 | 45.6 | 28.7 | 0.0082 | +| 4 | **fail** | `094917` | 575 | 79.2 | 23 | 2151.9 | 2130.2 | 334 | 46.7 | 32.3 | 0.0074 | +| 5 | pass | `095054` | 802 | 112.3 | 33 | 2172.7 | 2150.0 | 284 | 109.9 | 2.4 | 0.0049 | +| 6 | pass | `095305` | 705 | 99.0 | 29 | 2180.9 | 2158.6 | 338 | 98.8 | 0.1 | 0.0058 | +| 7 | pass | `095459` | 900 | 122.5 | 36 | 2138.4 | 2116.6 | 291 | 122.1 | 0.3 | 0.0041 | +| 8 | pass | `095720` | 900 | 125.6 | 36 | 2149.5 | 2127.3 | 295 | 123.8 | 0.2 | 0.0046 | +| 9 | pass | `095941` | 875 | 121.5 | 35 | 2159.5 | 2137.0 | 390 | 119.5 | 0.2 | 0.0045 | +| 10 | pass | `100204` | 1025 | 144.8 | 41 | 2146.2 | 2123.7 | 398 | 142.7 | 0.4 | 0.0037 | +| 11 | pass | `100446` | 877 | 123.6 | 36 | 2176.2 | 2153.6 | 313 | 121.1 | 2.5 | 0.0042 | +| 12 | **fail** | `100713` | 600 | 83.3 | 24 | 2189.2 | 2166.5 | 384 | 53.5 | 29.4 | 0.0063 | +| 13 | **fail** | `100858` | 580 | 81.4 | 24 | 2158.1 | 2135.9 | 363 | 50.2 | 31.2 | 0.0064 | +| 14 | pass | `101100` | 1025 | 146.8 | 41 | 2177.8 | 2155.2 | 354 | 145.1 | 0.5 | 0.0039 | +| 15 | **fail** | `101344` | 575 | 80.7 | 23 | 2135.6 | 2113.1 | 292 | 49.7 | 30.0 | 0.0062 | +| 16 | pass | `101525` | 766 | 108.4 | 31 | 2177.6 | 2154.7 | 307 | 106.7 | 1.7 | 0.0055 | +| 17 | pass | `101736` | 800 | 112.8 | 32 | 2168.5 | 2146.1 | 340 | 111.7 | 0.3 | 0.0051 | +| 18 | **fail** | `101952` | 725 | 105.6 | 29 | 2158.9 | 2136.6 | 323 | 72.2 | 32.4 | 0.0051 | +| 19 | **fail** | `102158` | 555 | 80.2 | 23 | 2211.4 | 2188.7 | 315 | 47.6 | 32.6 | 0.0070 | +| 20 | pass* | `102344` | 895 | 124.3 | 36 | 2169.4 | 2146.7 | 366 | 124.2 | 0.1 | 0.0043 | + +\* not in the supplied label list; scored from the video and the signals. + +## The failure mode + +Unchanged from `REPORT_SMOLVLA.md`: transport succeeds, release does not. In all +13 failures the last gripper command is a CLOSE issued long before the operator +stopped, the measured gripper reads 0.000, and the last frame shows the cup still +clamped in the gripper next to the basket. +`figures_vlacpp/end_side_gpu.jpg` and `figures_vlacpp/end_side_cpu.jpg` are 5x4 +contact sheets of the last frame of every episode, green border = pass. + +**End pose against the 44 training demonstrations.** Release pose = last frame of +a success, stuck pose = last frame of a failure; `z` is in demonstration sd: + +| Joint | Train release | sd | Release G / z | Release C / z | Stuck G / z | Stuck C / z | +|---|---|---|---|---|---|---| +| shoulder_pan | -1.0391 | 0.019 | -1.0580 / -1.0 | -1.0645 / -1.3 | -1.0806 / -2.2 | -1.1296 / **-4.8** | +| shoulder_lift | -1.9178 | 0.026 | -1.8723 / +1.8 | -1.8801 / +1.5 | -1.8591 / +2.3 | -1.8266 / **+3.6** | +| elbow | -1.5323 | 0.031 | -1.5754 / -1.4 | -1.5838 / -1.7 | -1.5369 / -0.1 | -1.5447 / -0.4 | +| wrist_1 | -1.2655 | 0.025 | -1.2683 / -0.1 | -1.2521 / +0.5 | -1.3191 / -2.1 | -1.3445 / **-3.2** | +| wrist_2 | +1.5713 | 0.0001 | +1.5713 | +1.5712 | +1.5713 | +1.5712 | +| wrist_3 | -2.6082 | 0.019 | -2.6284 / -1.1 | -2.6346 / -1.4 | -2.6468 / -2.0 | -2.6983 / **-4.7** | + +RMS z over the five live joints: release 1.21 (G) and 1.34 (C), stuck 1.94 (G) and +3.68 (C). Successful releases land inside the demonstrations' spread on both +backends; the stuck pose is outside it, and further out on CPU. `wrist_2` is dead, +as `DATASET.md` documents. + +The two backends' *release* poses agree to 0.016 rad (0.9 deg) - the policy stops +in the same place regardless of engine. The *stuck* poses agree only to 0.051 rad, +because the CPU stuck pose is more extreme; with n=5 and n=8 that is one cluster +being sampled differently, not a second failure mode. + +**The stall is an oscillation, and its size scales with the control rate.** Joint +travel over the last 20 s: + +| | Failures | Successes | +|---|---|---| +| CUDA | 1.81 - 2.36 rad (mean 2.10) | 0.83 - 2.04 rad (mean 1.51) | +| CPU | 1.05 - 1.22 rad (mean 1.14) | 0.27 - 0.52 rad (mean 0.35) | + +On both, the failures move more, and the motion is a coupled +`shoulder_pan` / `wrist_3` sweep of nearly equal amplitude (CPU failures: 0.45 and +0.45 rad of the 1.14 total). But **the separation only holds on CPU**. On CUDA the +ranges overlap - at 17.7 Hz a successful episode also covers 1.5 rad in its last +20 s simply because it is still executing 350 actions in that window. Travel is +not a rate-portable failure signal; the gripper-state detector is. + +## Latency and policy behaviour + +`REPORT_SMOLVLA.md` found that going from 1.18 s to 2.34 s per query cost **+19% +steps to first grasp** (384 -> 459, Welch t = 5.8) and read it as closed-loop +degradation. This pair does not reproduce it, at a much larger latency ratio: + +| Session | Engine | Round trip | Steps to first grasp | +|---|---|---|---| +| G, vla.cpp CUDA | CUDA | 111 ms | **335 +- 26** | +| C, vla.cpp CPU | CPU | 2164 ms | **329 +- 41** | +| | | | Welch t = 0.57, df = 32, n.s. | + +For context, all five sessions on this task: + +| Session | Server | Round trip | Steps to first grasp | Rate | +|---|---|---|---|---| +| `rollouts_vlacpp_gpu` | vla.cpp CUDA | 111 ms | 335 +- 26 | 17.68 Hz | +| `rollouts_m4` | vla.simd, M4, over LAN | 1177 ms | 384 +- 37 | 9.74 Hz | +| `rollouts_vlacpp_cpu1` | vla.cpp CPU | 2164 ms | 329 +- 41 | 7.11 Hz | +| `rollouts_vlacpp_cpu` | vla.cpp CPU, no FA | 2374 ms | 364 +- 44 | 6.69 Hz | +| `rollouts_ryzen7` | vla.simd, Ryzen, over LAN | 2343 ms | 459 +- 44 | 6.73 Hz | + +Steps-to-grasp is not monotone in latency: the two slowest sessions sit at 364 and +459, and the fastest sits at 335, between them. Three sessions at 2.1 - 2.4 s span +329 to 459 steps. Whatever produced the M4 -> Ryzen difference, it is not the +latency alone - cup placement and engine differ too - and the +19% figure should +not be carried forward as a latency law. **Within a clean single-variable +comparison, latency changes the wall clock and not the action count.** + +Steps per successful episode tell the same story: 835 on CUDA and 857 on CPU +(1.16x and 1.19x a 721-step demonstration), against 951 and 1073 in the earlier +sessions. Both vla.cpp sessions are closer to the demonstrations than either +vla.simd session, on either backend. + +## Infrastructure + +| Metric | CUDA | CPU | +|---|---|---| +| Episodes / steps / queries | 20 / 15,681 / 637 | 20 / 15,030 / 606 | +| Recorded wall time | 14.8 min | 35.2 min | +| Blocked on the server | 71 s of 887 s = 8.0% | 1312 s of 2113 s = 62.1% | +| Effective control rate | 17.68 Hz | 7.11 Hz | +| Dropped recording ticks | 0 | 0 | +| Ticks with step period > 100 ms | 4.31% (1 per chunk) | 4.22% (1 per chunk) | +| Slew-limited commands | 0.08% | 0.04% | +| Stop reason | `interrupted` x20 | `interrupted` x20 | + +Zero `server_error` stops across 1243 queries. The bridge held for 50 minutes of +continuous serving across two backends with no restart. Step period is 49.99 / +50.00 ms at p50 and 50.44 / 50.39 ms at p95 on the ticks that do not carry a +query, so the 20 Hz timer itself is exact - the entire rate loss is the blocking +query. + +## Caveats + +- **n=20 per backend, one scene, one operator, one day.** 75% and 60% have Wilson + intervals of 53-89% and 39-78%. Fisher p = 0.50. The sessions are consistent + with equal success rates and with a real 15-point gap; neither is established. +- **The CPU session has 19 operator labels for 20 rollouts.** Ep20's label comes + from the video and the signals, both unambiguous, but it is not the operator's. +- **Stop times are the operator's.** Episodes end on Ctrl+C, so step counts and + durations partly measure operator patience. "Failures are shorter" is close to + circular; the stall-then-abort structure is the content. +- **The two sessions ran back to back, CUDA first (08:51-09:13) then CPU + (09:44-10:23).** Any scene drift over those 90 minutes is confounded with the + backend. The cup start pose in the contact sheets looks stable, but it was not + measured. +- **Latency is only compared at two points.** With n=2 backends, "no effect on + steps-to-grasp" is a failure to reproduce, not a demonstration of no effect. +- **Numerics are not compared here.** `RUN_SMOLVLA.md` covers that: CUDA bf16 is + 6.3e-04 rad from torch fp32, CPU f32 is 1.6e-05. Both are far tighter than the + torch GPU server this robot has been running, so neither session's behaviour + should be attributed to engine arithmetic. +- Nothing here measures grasp force or whether the cup was held securely. + +## Recommendations + +1. **Make vla.cpp/CUDA the default server on this host.** 111 ms per query, 1.1 GB + VRAM, 17.7 Hz on the robot, stable across a full session. It is 2x faster than + the torch server for half the VRAM, three orders of magnitude closer to torch + fp32, and the first configuration that gets near the trained 20 Hz. +2. **Keep `VLA_SMOLVLA_FA=1` in the CPU command.** -8.8% engine time and a 4x + reduction in run-to-run spread, measured on the robot. +3. **CPU is a fallback, and vla.simd is the better one.** 2.16 s per query against + vla.simd's ~1.0 s. Use the vla.cpp CPU path when the box has no GPU *and* the + single-runtime property matters more than 2x latency. +4. **The next latency win on CUDA is the bridge, not the engine.** 21 ms of + resize-with-pad and tokenization is 19% of the query. The tokenization is + constant per session and already cacheable (`--tokens`); the two resizes are + the rest. Moving them into the engine, or into the client where the frames are + already in memory, would take the query under 95 ms. +5. **Prefetching is now worth it on CUDA and only on CUDA.** At 111 ms of + inference against 1250 ms of execution per chunk, a background thread would + remove the dwell entirely and reach the full 20 Hz. On CPU, prefetching alone + does not help - 2.16 s does not fit in 1.25 s - it needs `--exec-horizon 50` + too. +6. **Ship the stall detector.** 39/39 on this pair, 29/29 failures and 0 false + alarms over 79 episodes and four backends. Count the idle in policy steps, not + seconds, so the threshold survives the higher control rate. +7. **The release is still the bug, and speed does not fix it.** 13 failures across + both sessions, all at the basket, all with grasp and transport intact, at 17.7 + Hz and at 7.1 Hz alike. The fix is demonstrations that release from a range of + poses around the basket - the gap `DATASET.md` calls out as "no failures and no + recoveries" - not a faster server. + +## Reproducing these numbers + +```bash +python3 - <<'EOF' +import json, numpy as np +from pathlib import Path +for root in ("rollouts_vlacpp_gpu", "rollouts_vlacpp_cpu1"): + lat, inf, hd, steps, wall = [], [], [], 0, 0.0 + for d in sorted(Path(root).glob("rollout_*")): + m = json.loads((d/"metadata.json").read_text()) + z = np.load(d/"states.npz") + lat.append(z["query_latency_ms"]); inf.append(z["query_inference_ms"]) + hd.append(z["query_handling_ms"]) + steps += m["steps"]; wall += m["duration_s"] + sc, t, g = z["sent_close"], z["t"], z["state_grip"] + fl = np.where(np.diff(sc.astype(int)) != 0)[0] + idle = t[-1] - t[fl[-1]] + print(root, d.name[8:], m["steps"], + "rt %.0fms" % m["timings_ms"]["server_roundtrip"]["mean"], + "eng %.0fms" % m["timings_ms"]["server_inference"]["mean"], + "grasp@%d" % z["step"][fl[0]], "idle %.0fs" % idle, + "STALLED" if (g[-1] < 0.5 and idle > 10) else "ok") + lat, inf, hd = map(np.concatenate, (lat, inf, hd)) + print(f"{root}: rt {lat.mean():.1f} (p95 {np.percentile(lat,95):.1f}) " + f"engine {inf.mean():.1f} bridge {(hd-inf).mean():.1f} " + f"loopback {(lat-hd).mean():.1f} blocked {100*lat.sum()/1000/wall:.1f}% " + f"rate {steps/wall:.2f} Hz\n") +EOF +``` + +Figures in `figures_vlacpp/` (the rollout directories are owned by another user +and not writable): `end_side_gpu.jpg`, `end_side_cpu.jpg` (5x4 contact sheets of +every last frame, green border = pass) and +`end_cpu_ep18_fail_vs_ep20_pass.jpg`. diff --git a/docs/corl-paper/solver_sweep.pdf b/docs/corl-paper/solver_sweep.pdf new file mode 100644 index 0000000000000000000000000000000000000000..984b9d264bd0024fd333b7e7918daa007a35f9d3 GIT binary patch literal 20987 zcma&Mb9AKL_AT5Uci2ffR>e-mwr$(CZQHi(bgWLtw$ZU|-t>FU8TX8P?>Bzms6U=( zbIrB(t{QXARm8G_!qjv$jL^iDYk-P2XgUBbz((I3nv)X%RCKd70ssZ{9Q7=1OaMSx zJrg4b0R86#c>p&zw2`&p--)dMwLsX$+7ZC;&jg^VjJ~;%fg^zBUrixbM-c@_Jx3z| zmp%}S30)S#xpAh)} z{S^53Qw*T>UvTLDZwLP3{)^t;=o8()-~;829BiEI4L<$(i~rNUl#!vC9>0z2r$yS& z4-*|T4J$o89SbuvBLf?No}Q70m6`F=5>VjN$Ip!%0Brw4%4cnD^Vy>NyXwE=@^|<@ zfs{0|HgPls(EZ!Eu$kqj>j0pzbvb7-XAQovg*I_mYsv;SN!1} zSd;QpBaVTX`Blz1C#U(W;eMBcwq?Kzn?03d^B=B#kDPEaZ9MPd#${$teYKLKNNDjmA(CG^h*8Bj$QrJ zw_-Jr@)t#;JiVkvrszv=(}+2Eb?C-~II&eIM{iLyBpL>}rtI{gSsaEHPY-iy6|eA0 z1n+_dCE;0M8XJLeFH}X-O&Z6rsfYg02}`&J-qgV~U*h8|TA0B;$ufW=EyPHc=B3~& zLIa#JWh!|Q*w5m*3E}n(N(;?s_2_OQz7ZjGYU5cxR}>pnzW`fd_U}ER7|1)|8!qXC zF72}^RcJq$;*?fWz1uj(Gv>AfmBHRXe_(3RC`6#H=x3m#Xklw`7ot-3P=W1aIT?&T z^+?6RPbgeETee~jh>~%+lD!K1TW5-RnCNOGST6RVN2w(~q|IF@MD%j{YGJ%CC2yTg zZKGhoeJQz{J184=zAhM<;*#o&*4;KEsR3g%>7(jT9cov5q}Col2^3nrkBtUY1dwCy zhzWe@+=s{X;0%d1CYa&Bl8z+S#PCAsfVTct8!v-HSNYuz#{=>`;1{%EUEE;s7$)mi z@Vf~>#^DQZvcm~JOP7t zJzVkwVBtph^Y@Plm-Lc}v*oF6V!1d_P34j&FU{ftQLe29O#%6>N_@{1du^2@g`I^1 zOt<1cx_ETPmJaQY&XQ_tmN-sk78JNQoF@6?Zu7#LaH>xLl&?lcOheTx%K(1juKm_;ZZqFcz|vD zz%;h{d_G-OK>k~M&uWSs%QquXVvC_DY)k}SQ~!2?CBO2H51F6j#ZYXgC?HIZ76yn{GHq}>7txpiN#pml!9eLUT#*J3`OZQ zR$i_@R#L73zcrt}%$_Sr-z$9+q)krV(ROLb?5h_D1B!!vuMNK6S5~l}V4KLD_n8dF zhORItt@ha1DuTu1_?3)vk(@39!9^j_{2@+)L~jGdeN$NUd8sb+m=(fI$=u#5Rii`MfrYZ6S@{5f!hGe)3W32sA1NPwZ>+MYMVPr+b2! zvlXEcOXs^f^)K(^QZ@yxhC=4%MaSo6qGwVy-c9ow7=J4F0${_4tsRpy92S>wQm6NY z*C!$|w~TwLReiplqT@Vj*ANaCOBQg)cPMJ%;F~RG&m6BQ9}qLmc|iId=cVxDQ3(HP zu|lzjK&y&hPK*3gL54hBe$+@@AZ0?iytQn^c4A{L#r{HI;*)-1!b)q{EQn#%O<4kG zY@s4e2Su&7D%=-`>*(SA<2IIL1>G&*tQXlz=5`^iD!1fGTZ-~R7K+kg%IH~c3dVEh zWXM>Cntmnwu!=!L0O1?^^7pI=^`L<2`53=!IdOouMG8lL*iSrntTE*&h95z2s2ZAS zpunXV|LmPd#L7{GEz*Clopk3LD0B^jdJ|W9aJDY^VY&UV@8z)3VLg$~ zlTqy!F{yHNfWpmXrCkV#3`}P^jLR15Ft2{f6C~-o7YN%+ZIG3?!3xo^JDazsn<(jP zAM6i0I`*6e4WDO7II`+r=-q}V1$t2&ZO7UcD<}j93_d_}Y-C^fcy*xI@?l?>^C=#btxCQT3w9QD zWw*^San#i89}Qqdy;&Z8Z)Y`xoWm=dcTv~!7ua|EV51mLOtu|512*%wJCXB5^go5s zY)Y9K6%|}p#uuW+-~AqzaE2B!6oN{RNu;^CA)X0!U~W@9HzXw;@_9~;MN!VFJ=W_d zhunSJ(35N#{`$~@1*8RRYcN^tqsOA9SJbmXj1bnzGwUT7=)%gg#=J50`Ce?v13swG zD+dn*fOFQDxx7bi!@6knNN_1+z8cP(gL zz5E0$DWh|PetiJO(i{`4gK8cK zDJp}0W8YF7m=73`Ody5fmX}meW9$q+_Q4j`#EQnmS9AhmcCX^wn|Uz`DF}k&CHblH zAPtn@X%l}hlv+?SZNhRP)FgbJ^1*dlQBdkC46{U!BrGXdNh?e(DH}uo8kO+mV#UiH z_W^VpupdpQ!J#g3ce|UX$6LFMo0~DE)D^sZr8!f?NNH#Q7g*J}(P~7gglSG2ltfV= zF=NCB4N8v18e8A5l5<08@vOJ{9fEo5;f`c(clO@l>?oL zR>FFwBSQV_?Qd%GZ!-PwB;_CG{g;i?v$E3vCv*9Svj6SW0BHTEkN&^Yhkwy_IsgOx zC-n#Zo5a)8(XxC}_y2*yk1Kh|DlTGtoWEuD(ZIm8pZ0$tpz?!5ax=QZDMZs0oTEV0 zr%QsW=iF@t;U-fPz>P-m%K%CaZFha4QI3n}@ zj}4yBb-6Zef*m=fqh;Z_WJHw0H&;=6+`|z!lkv<8%gR z#uo8F&pq(efqi^>p0*|?5@u=KwVYQSJhiY9I*GnWe;`J0fh?sUg&#Xopg$}KKT<+~ zYeVP(*aAV9gG6p(e9{W5h5ZF`J{l*yJG1zg{e1&9n1dxqh+n^9 z6ons+oi~LK%m6IPODXsn821vZw4PSIuqAzlm5l|1B7eZZ&`&C>rXee-VO+7Yv0HH| zP40Rq?ad#_kij4Cd&Nw*uhld(hO3*xoE7UO^W<|&_^2(7PozH-4R=Z#h@;*pHc+~q ziWD_3$$YKx`u!?A(APMu-W~9*BIj3NTb80?t)iY9>?}Gi3J9`7j?o0o5{)oMQ zaB6e1wLMVpg7OfgvxZZ|BoXUJG3cCCl27t>EDLK2>I#vK>KpQkt`M^m^|f%|K%Ba3 z4%*U78hK}YQKb+yW|H|vUU{T0n}v&9MCo3(*`04wcNtfyIo@E-22iRLYmO?eEYcR6 zVF2s}FON9WhR~!flV^Smuh|sccn^8cNL^0@5wS{e>6fNc(z9%CW_YSl(+^=357qHD z&_?yvVYx^9E)0Bd#i07mYPoq8TUnJWfTjjE6(PCTuy08mgZz9wqTD&C6n1XG*FxKrK z_RCIWgtIi5TIhaWDu_2}pj|(37`%JByyw5_L}WFb;i<|URbW|IuoxqIDDSy)y z3f-_c0EDWkHpHnh3FB_8$^a;ZIy+i_v^lEH?g~q1g8Z^R#tB??v0r1iILq*+s3OZ~ zUD(5Jw_e#_W!-pDJ)!bq8O>5w&h!9bJLuF&Hd_Lw>0j4+CVxGa`7IjQIuog9&(k7BDzL-)D4EE4IKZcct$Wm?e~O2m@&X5YS`i1jgMNtia{s*Bg?T#L zuRUQ=gW%!ZzL)*u6eFXfP$qoc`gD7$-QSfPZ*sh+LBN#+`mHzjl;zSfLxMXiH0uv~ z>uiiIZ1!VjeGctq9kpKb~=>>%%#)_Qn)$vrCOv-gq?4oC!nRwmt3GH16M76 zBY*{S*Rhyzrn-89NTxbft;!T#r60;*a__lX$q0I_ct*Cv^a#oQd5TU$GlF2r?uNN2 z5)zvBWst=Ea`nH1^tt3wMfhR@trzO#Jx!p52t-i5&?KOlS`CSz@tlVE;ey&+=luF3 zws*HNsnidXBerX&qQIrcSZHmkNe(blHMZg$n0`Fs)c1gX5XQ}1`1%}s;)V%&Yx|EO z0Dvso`^12Y_#r8~p*7dt`-se;+%XLPdu%D=n%~$Pzp>#Q9It3g-fh3DHQX2}(-}MH zH{8SxD-`>Mp!OX|#&%&E77h+aiWfGhxns{bUFG7KZGfdIgW8m9xFc_M63MfvmSdH@ z5%l~0&>9mLv8OUN29VR~kSGzvb&~88_V$2c-+l#U&m?@B%3n}EZLyfr7(=tl-2_xwwlHFXDykRGaiocavY(ppfo zI}%EuKI>MXmLU$*xtu*$(I1w~K-_lmtI@=VRl$c7$Vbyv_ht82degv_|C_te|5XQf;GpVz+6M}|B5r8(e8OL_zeg8wQzbXDutF5Yq1=&`dxc|C zWaOiY?(2MrIXNeFXU2PqmU()Av8Z;? z`M-lj_um@#zeMqWly0DclfL7>G;%S)zde7)=m9`}JqM$IO%N3jR8&)^6f`o|Q+84? zvNtoP7W-7w%?zM{0%C#+ZVrw{R$|u1Hh{l!yS$N!*(Ysv1Ca53D(gn%0HBP$;pdu9 zRh&%B@KbX(b9DRcRB*DjwKTH&D`wOFm5qUaO$iz~7}%THI@;I+=>Dy^t9+`~pEKk? zC2KK3MH>+@K`A|3z(2R?zjbQ5{}!tMpU(a5r2YTtm5}wPn*A3s@IQ#DMFGIS=suaa zp0(p&tPYcbKdba<-b_7t< zGcf51nVFb=h8-HH`k&WNNB=ocK5G+8qt9>y6+Vwt`R~hLS3lE)&kONSQt^2X zQhKf`W`>`I0A^;E|0-Ghb4>rrLs%I%UTx|Gs_yNlECK>1aQ{t^X-3(EeTX zKPM#~Zn{5&R$P=+TqY$a=N*jVhv^v7hy>!r=p~`>zvs3B{h&g>vmo(G2< z4?fuC+3BccZx`X?P-W>`&$nza#wxLK+ZI#Z@QYXn zRBF5BGCp5d(xav}nVa=D{B4xqRj~Ng(IPi#ZJd?B<$L`af&4lwHI?o(;nE>bO(BTl zYi!-7&Uxp+5>(D7jxEC56iMS_?EwKSIf8w>-vb=`OkSRB7I_>A!ZBz!e;nke_9A5vfvA z|4u}i%!q;=aX~*Tz(o*P<>U+!;^PUJRxKit6-3W~qh=BH07OcBS!S4OXJV!}%+Q4Z z#Vm}B6i}OrQOG`pzj|V8i?o;h317yo6zT>hcm&SpjgD0D;OzHsd-W>&lC3i}GReL4 z<%Z^Ib1-jhOI3Z-@*#ca=S^$pvx9Y0+E=JIMTE1{jOP2V?|nbY*r#m4MSc(`dN4Ek zf9cZAG^X<`>3if(XK%svMV(<>W2U*;Z+T}KG+M)kEUbm^z}OejWN!v-qw7u26mh^{ zkR7}U++hp;W?r^PYI0u6PlS(FzF$?3a-m4wlVXh}z)uVfPQ8zQTRUSS2Qz193zRSb z`GIhs3Vmzjw-8bwXNuhh!^jDj^rbi@-<%g$UiglWT(<$#Jt{r2ICr@emWx-!N$2>z z@@4j%$3^Gxz2(L8;bq;Vee=w_{RLs&#u@L7d*)62;m<1DV@S@{^Xd+y)W~c=&^0n1 zMP)uY6z2CPeXvHkRsL|M7Zf*0F33YPRsmlhutZ20QB+qbRG37$JP^C@mnc-sWG+F_ zP4&Zyhq9-##k_e}-o$Frwq|d6;H!1YrJ{nCPR!HpW8;ZS}SWQyd0PF|p@gKtT%e;=q7Ix=Rj47U=}MCecd|SgM?^-y3EdH~s)UgIxnYF=nJ>UO zhy+2HBVlQ@J&5vC+-1`CO|O`v|DmB;La|j{PP(FN#i^y8!vq3rc-(R?UtO%DEh*~3 z^D2hMJ8nB8mZ5L{d7(5ePfcVmCeT5_gg4=Si!l?ADV`tQ6m&db9Ff<_=z&;?KvWf` z^acPl?cf!c0c^0^V280KEmeI89cFmZsz!%P1EsmHwj3*4M zuKAhc`ewe`TCv>}Q=K-ZcDaZacIJ1oY&AlDm5wJW7q;foI2PGo*D;QnjC+_f4a{>{ z#e!;3bCa>;7w&#fc08@SJ>GFK9wSfpp3G!3{OELmE|CGII!r2vL~}3O@0wZ~m{=R^ zDhx(5vvk`BuC~2gRdst=ul_t}iI zS!=oKo0zX~87V>Cf^mhX`HhD9BXNjw*JMxYIvnYku(@Y!G%1acI15_2Q#@dykiU|T za}l*=g`tkPpnI8!ej}1r-D>Z-a2hv6V#_4wL3wTxQOj zw||HX06EMVpFnMBFu50k9JZ9|f^S6Y7|_dC0uc)63gkn1UHucs%K8X0S|c=r`c zfH0I!7nN%)enHaM48R%o4|nv8YgJ9bljbeL+$e<$ajQ`d52jp*)`q4n;9g0G^GywUG|J*A3>T`Z$h2M!)F_l6x zh$!mikUXcfDN6CUD++p?t~~XOa}Kud2sW+4OiL_GiP;PoGs1vr65Tp!fI)0wq@78N zl%}v}+ptw1?$`+4R{dQnwcf|;N`VT8`41{f-y%aKS&SeOaqBD;j5||Ag+sFK@H+`9 z<<5y{Fw28^2)^IOR*kMWuwm)(l`wZ)C>$q3Ky-T z<5@;22)ieO9_qV+*ET_kO%sC{so5alqAY!wsMy9z^t14F7N-VJq<~nhtmI(swGfZ# zGH~0taQ`V>1BG9 z*LgcjPtl6L0d{<-0dNMbH-7qJ>5+ct7DyhuG;5 zpox;WppOMGk~AI{ue0PS$#h4~o`V*qq76zXMP!&=IV0y0q6;UU05Ns=$p7i{`&KL-9>tE`)D3??O|n>2Gb z9X#!?IbqDUBDf6CZg20Nq;w~IgqZs%^m)`|aZ^U=R?BTR!W+)yJ{G$&iiFSOcMXMx zTTaxcn#;%;)=~5H68{cF+_0Rz!lcp6_(IU?KC0makwQ<~>KY|oz*0v*dYwN~`16p; zeHwr%G;Jt8M|@1mM%qf!I_`GG<=Ce$?4Ei}COv@z)v(H0hK`MQqo`$Ipyeg7LrJJZ z^#Y6hG?GN~ zn2+={x%Ecw;)Fk$5gaU9XZNRYsF*yQ4wV>XaQ9{Bmy_{C#g6=UJhL6Q8jX{Z0k_UU zRu9c|1llsySpV$O;={ik74)d){b9n~gO|iS1`d3l_^#FI%tW?49cv zmUof&)ho20JYGA$g0N(E29n=iG_p6kbEwo-ms*^)40bf`8OD9u9FJtiO-|!p7H~N| z?zdLF6crFcT*JOd?TOh9{h=U8rgv!S91_JA!n9@nrraCpicr!ZbhA&JeO!*?`t>=$ z^GkGDK;BBYqdw^lk@Uhan?a^@$>@>u0DJ>%1AN2LP3>1lU;5{uWE|*LSaAFO4L&F~ zZv6}Y?xFQ1?;;qiy`5{)+zjY9LKtbhlo*?pF0a|*v#7BI}lwk#Z zed`}de_WYS=N>jRa=EuGJ3PU42ne#IavLnVum#UBXcjws91S@HxdJ+8ka61E zv`;}5qXIgoGT0vA>uhq{Vsbf2A`{%+3_6Iz`m}Yv*+Jz)cJb+13KfvcJM(FKv?kpJ z+efR5DhFt`a`TEzj8LW3+)`t$mls_S&T7^Mu?IDNaoI_We7jka)dXBtnMr zG2TKsBc&Lditi{jWfKCvoC#xTRL4B}Kt5}OuKwDe=1^`IrrcOq*+Wa@)jj9Bai1wX zIyV1awnGcC7QR*&4D7SLVt8kDPPJ57mHPg^hb=cQD*XXQRR+@b!BbRmAs z)FX+NAUf4Fx^ip>%|1a_HNAjS8*CLREESxvY{;!d5Mg2_ddLzUx0z9Xy(tsLNTV?^jfiKaI_Vt@T6a(8sQ< zh|iW>%gG8sYN9&NoX#L0Utqslz^>Gq*F%$Np}1sh*vIvqCgU!>h1PJU4Tmcnts7nl&1?|_@ck+M zqo-0-HQEkA-@|uA{sJ7P3{JP)yQA#L?kN~#YBt9}`}|SU`vz-&U^a?F-G8aW@z~H~ zv4j|ZgnpBrrA~{}4N0CTl;5n}a8jN>>eRvyz_Cx|ek+~Y z!;UW;)$0u78(z3=XxSD<(~oU~N%>^!D3=2hvYSjvNCD0Ll52rh`_jzg^B}JRcW=Gf z;nwcy_cUU^`NK`|TUX2bd369Z_9jQ2gAc!#KmgK9C)CCxWselWy?EbOa64pT|ApsK05Db3Sy8GAFQbJM1EQ-es+EK`MkAyNXugu=kU zGhf=c{h-bOe%s|9l~^fIBRWMsM`Hhm;e^I&Lmzu-d!g$QLz)ZaA-cTz^4~aO;%jk% z@$TEvLSBJ_)LQY#Sz*^c6Ib>O6bBu<BWI%*qh@2A zq8*|fVj{B0$L1#H#^)v-#~mjlsizaa8LiN3m37Uj?Nd0U^rL#12dP_CRV(XQl{zi7 znXfn1&1a7#?=BcZv`(sA&Q z2WaP?`+=O*$r4pbgIQ#HH9EadUw-E}hD(iuYg*}w@E0#YauF2?wKXAbt=mccR(9&_ z?;KV@Q^={Ej<2bpqeaBFFZq4!1y6Ww&L(21Dz8u500p)ZK%3vuIJ)Tlst%T73XK`B z6ONoPl8}7ImiBw$mMu(%o>UBLb)f0W>gUf|=9l5i1VmDt?_j%sA~W!#h3JSdQKzVb z9bIG&)A9@{h|b!bDi&$V%Q-qEHkew996vl3X>^TG%7bD;8?JS--;Y>aXV4_fNAQ*d zW1QZ~3e8^+4XC@>7^~ft^m?Cu|BgsJ;p@56+kk$yFQAj}$Y5B83STaU-!$%FDnyz` zzn;e$kAt@g4!T;FtVDY+Cxi^Vx|Bx&P>UA3QFS6cQK$507QkP|w@NO><@U~F?bB@&U zxT~S#eSLIblat$$uO1FT%_bYX%gb?kC)lm^&{C$|5VtFWDenkx=yWL-=XmDI(&FwY z53=X{;df1T91wtrDs}3S@g+8SSlHuZ#$D|llY<8BEw6M=`QdOUY8C6VLcNve0l(#l z;k46b=Kb^%McMuId@_nagP^xBFY2C96iBNKb|hpu%h6mW$%Go4O`+2rNB3e$aHWqo z5^4`6rs%ndZ#8P&E#Mgs;U+itI2YJIyc>cr_`&zlJfeHbxMr-JA9*(A@~?r1Z=q6{R4y53tVQ9rG@ z2AurG*0Ix{yTflc!dl*m>IM$EqNlVZLUKj|L^sy5iHh;tOyNcil11U^B1U9Z5g%$?>+t*g`_z}-^> zW%7juIG@Y5_x018h&I7A5KilP6>PkgXE$$ zN^WvK5s0+MF{m{r&5*pm^KKE>ptXK-y9;!n(kdh_4C?krEtSTEDRYW4V)G$fzj!9b zq~+12?)&ih3K|e^!d00b^>X$}Q<&_OB^*fggIgI7=YhDoRiuCcS1X$5S*j-;iyExMi?J zT96^nNM4SfLpgFNgz*ht$yilhoVHBX6V`O+c-|R0tJ@RDAHmjG8IVX(TrFxP1rDW` zfRqirJ})mC6KZ|6olgt{uY5z1oUt}1)p3CCb#Di z487ntB0M8`g6YwH`LqO#Dema#5d!NED-=DSb!oaBm7Z9_=dVngs89X%VrAQmQACcBmHM9y)j2(qN z%NFd2Q`M*j+C+B4>Q;VwBe*n@A@Nf0y%CXo%S+os@3cU`xa>U5ezTM1D8)kDr+m$6 z?yM^{N8^RI>3|QgNbU;{7f-~XQ&4Rv2M0PaT|vM-0R#ibL?VS!Sd~ci=P&#s5<8?p z4!hCRd@|y#s$}|3kQDeDeC%Dz!sD)uxo@v}OO+EDN};YP)WH-cb#-oAk*qLZ7$22Q z3m0As72s6Ior1|YiqM1*eYxV9oaf~!Mq`r+?A>GmlZrKk${qninJs_+0V-@}kM{+u zaTaGP&ri+ox@Vu)!na+>T_@*iRF*Arul7nL#Q?(}r()ZZ1ImCmEnn?aSy z)mFqk;Oj2p$yBAnm3!8?!!BD00%XR~zjH_t`G35EA%Bk8cljPfxg#Z~ z1t+Z?4lN?<=$L5qGMowsH}690$R~~jC?Fqo1Ayv++MPV-#0#ZaV=5#l%QBFQ-lolKXH9O!Kpo>~N@m2rLhiEYhM{cAdUwV`@XAID z2@?^lr#FV_+!c>>F z%

#4gg-FZ30Q<`nBF7MtJhOhq?TIZP;~80x6EgnV*|mncs6z`MJdIm&?>p8qvPQ zS36NMFYNrzByR2~^%i%VrP-gTkXr}J&#C$mxcZ77Rs}n$KL|1JMvn{*l-i80b|z;B zvrngs3}YBNq-#b`?g!ktTKTJ)RZ=P@o%Y-8*IVltI4iO$v?|ss3{*TYGvTTsTm&ow zOqv}&FO7%GskKaFVh4d+j#G}6&Jy=a*QFOCX(g#8=`ZOenZTS-+Nnq&;oE(r%r{j_ zp3%or(i#fAlLel~p-P;Jg?aDW<3>UU@uj8+H0p7!o27fD%?^edtEs3WiEHEN>}J!EL_`vQjdL3k79QWh(-cP2LF@WIg#@%HP5HK)l+ z>WY)>5=ImMoEE5MNicP){uQYXX&WPDvtPZkatS0kLfjm(r+uJs+|v~j1sWW4Pn2}+ z{H%8m;<7V06&V6~Ykt`HBM8-@mQjY*1j~7VWxf#bxYP5bC2~56scI?QeF08`nhu7y zw3)Zbvw?<0+Mqk#^eAdZiqgx|3)J)F%jV6ch_Hs28JY2MiR;8P<4MFs>0x=j z?Wiyd@C;he#%ad{Zr~)Ah}Y!3)7;aH(`qI0z-65bj7nsYD{0STD=My%xL7erS$3X_ z?3oU08Y#0|Rn=G6kjO7|@!wf5E&HuzyjF}O<_Q=rz$g_38bLG{Wv$hI(T-CzL9-B6 zDpybmRH9~SlcuBJeDBm|nI_Q9XFhWKHHWS?SE!^xZ7f$a3k^Vh#te*OSmLD!m#xQF zD$uX{5NHK+>9x-K%C;9G$L7=r9o`2PLJa>B$erxSsXA02`FkhGF4-gKg*vE3mFy=K zNYb3@0*oG~ydG0Q2zY=(1!G6<3^emL} zb{q}v4EpPe>-OuCrx)7g;3xF#Md360_Ac4Vv7Mrg!)c{B`Z(9==hKmuzof}=D~Lib zd(z}gs_a$VFoE;uIae3;q~rZWr|z2{v#-yr+An`HSBg@%SGGuoN?Pe}BYq|hCLSo- zmYa)xF>HNF9Aold~F^EBXg#JN2)N3lbQAgy(h6a`X}yGU-d7*&f~OeCQU zmx}wiG$HLFlYdIei(zJ)ZtU;{g0xH3{gbzi#sqfpoe`TuYZj*H#R ztWEHXKBaP}+jSJ>}3+TeIO!25qo$k`<;B)cHGFD71xNI*i!N-XoJGP?1gzOdBd3uBvgl4Bkfxk7%V z@XlTiFtgv_+@QtyFYe+rwhY|GBc~U zS4G*WQ=mP~1WbIW&}dOtIoS`Ig;F^xjf+ayfFJy&cBL7W>}?thOQu(4k=j)VW(1bp z2_sl$4<7#0d|R3t9B<(K%X^|H!!_2{8={zWG}P6c)x^=AD5UO*^Yfr0rcQVFv{*=E zS?w&AD{qCx&VwZTQkbYNZz=Jv&E7Bbogv2doY~%!#W+E=`yi9}6E`{^0fhA-gHT2; z26c($5sIr(Ozuu_<&9~jJE2o7YN~MbwgF0v-n1{U`4bU+Roo)z;|aVExARfX&jz{I zlrHKuVDiIbLMG`AdpX1sEN;*JQRMpyn$0yQ;wMigDh4&4_(StLZWe=%>F7MyhI*4- zitid$qHZjhW#C)J-&e@`OP=IA{1UyPzGohdUJk&$f`2jjfbFr$Q|6FbDQW=(GguGW zn!A7S!J-M;ZBj`Or~7;x44Hx8cwq-keBwZHCe8owiyto0!n<9Gfw7*UBM&#fm}6se zCT&E~(s^Wq;#Ds|lwbd1b|*3)@2WuSPmnsM?H?EDTT?Tc@%6ev7N2j6tc;7hndK{X zAE_Fj8ioP#voS5UP68%5WB)@tF{;?DfmGB~$$-|d+St9jL6N^tx-2Bf`-c)^cO-&+ z-bo)vQT*%TiObd=8u7LA>KPP&-a}FH8t;>bC?eIBLs_ z)0z`u5Oq!BTq#_^`}91&I;ZwKOgR-0ihg>0p9@fgv&EgLyMVmF$K&NdQi_vggqkp^ zuV~|Vgc*{uP&HP!0ZQUjwR+JG-s;Mk0?m2%HA@n~W*{UDy(mqUL>jN;uZ z1yL%1l%@0k@s*mNW)5=P?{zH*WC%foy%AbB5C$%UElb|$WE1URI3yKh&5lNkxw$HHU_1toAW%3}coU;hMmIGTjsX(4pW>T@Y)!wN33nv#2LD0s-dK=8$ zYq#`+z{rR3+7m|Q5<X0%TQ?gb8In-1I^TyvZ^+D++pXD~90{l04_!-G z*H^pRU@2agyf9Px5k8jy&*FR!UnJ3)DM{D!8UQm%YCrZqT_MpS5ql7msw}m(Dz@k( zCT_V%;Bj1@yJ9IWPG!iG8JBxYcZoE@{2M$2Wzpnj;xHEKwYvuPgFC*Pmj3GYkcGSb zb7BgImZ-Gb@Gz&nJ{wyxu}IVI5T)|DsBecpaR}({13ep2LW+Rvfkx@_n zrYXT`md%@uG@MXYXl4bwNR&Y89KMD;ZCE1e%!t{7b5)2qz=SDU@Y^~Bt65|CKFKn` zk5Tk==gM%k=538R&C+Y^msJ#LjMcP5jM|VvNF9;c5_JZ+bwsu$TVxz#^g31SIO+29 z^`7RNT>X(Ab@Q+wvzR4L<1s{h6jro7mQlT)7jE+(vgO)%t!|oZjU15$iBtAm3oTBy z_JQA3QTP@JE;)&9%SW#e(`ho*vw&J8WXlOOQM8evA$s~b^5N@$Ot8e9n7JnFBk%X z){#Mx%n5RlJyGERLJ`%LxFW;fwsBZ`_gJlp3uv;mfL$fXoNB$(2qzPZ=(Mojx{DNB14?hHe*nd z6A7z=#9D>*V1kgQB+MZQc~NZP8-hc6r~@$;VJv!B)O{=&VOS(#G!)N4syKn}0u;}; zlrS7z^AxdEQA=e?6G~#`h@b}bFeQCzRQ+JP{es_M+f2iPf#TTGk-Xh?e&}22d?fCv zym->LNEHcTsO;)yyyS?wk+qqo&|&IE+jIY`oGT4#>WaePXe~j&tqWD^QwxGX@|L^} z!8R-bMJz&5M?=S81fqdpfI^v~R12brL#ejZ1y^uJtV>IEA}U4cs93bpnYu7l(NVz` zMJyl=XwQ8lWW&;#)?d9dN!~l}+~wSR&pi+3`_9~54IK;G!rw zmN`{M)~s3Hw9jvA&-2#dz11E!x39qVJ*}?vtSGdFAd?8xYlOlYQJ;WV9Tf9fcHx}! zszoDmFG*xFRn$hAtY)q2#;%q8=664<&GoM0^hXQI+wLWXFYvnDoXTFw_gLto8L#kf znbx`MdHU@g{ug{B^Xl^F`!DzC?v~Z%db{%M1OLqS;w0IOkci3pvfT+c>qD<`_U%uG zp0OP^H2`*9iW7qmJ2v`!@@jnK@yD-CFJF^C@`vO-o>7}V)HMf{1;hm8oCppplsWTl zU34k^`MO{^^_^sEdX|5{b`87k&)+(Z?V3}$joxsew_ZKkY1_0ds)b6w0MfDgX_U*1 z`(ISF_%+;7HBP_!Fyv`d{qM~lhqK7+AeXfxCuOJRDH1Nmb)@gdVvZ`GlsR6VqbWWu zdvj^F{;RCaZzS(D6dj-L`Y_{mWv99;|DqZ@?Bsts>=>Kowlw0vk%XC@2aoy1x!w@Qe6SBmZ`rQ;&nrL9f!Qx<|EdxRZN|-ckD?aP^km(c&igX8R*K<0I|r zNHsl4n(IYC8L^IwS}4&JIMnb|X;*e+?kgO|v6 zQ{c=!CGRPROc_Q!4 zhVO&Qn-J$DNK>4TKV zg%8})=$7IPic$_LG62wj(I*z88K_}=47a?{00P-F=m|^F=#qzaSeX4-_(lpmlCnZo zA@dK5s&T-dVBmQyT%oEog)tm$fb8=uG(qz)6c{qifF}CoNie4GnvqJuMleL+8XyEe z^aJ<2lmlHU#^9t-Cl=O23T;VQ4l)r@Fat;~g*Iu718d+pa0;L=xHn@8m~(pMBqlIB zfOa4!FkxStZz|*##yuq?1o}yz6PxlhNrGgcBw4J)2!UoIBVVj02KIqTk$Rybocsng zGya3wNn-LuGeNs?LX*kYI&;zPuv@l;hlJN&AV<( zatag6$9!BKKE*aL3-k4#O$5aaJ`Ysn*;`~}+pPw=fr_ykqgSNFJC`^pTPBsbDipI7 zc9MId_yNXgwmA&>$o{7bX66w8ej;Apf+5P8<-Y1m8z%`@s|4Z~O0Xb=ydZ5l5Uqr> z#SjT7^W+ObkPHpc**qhZX(<|QnnoRmeME@~T+I7miD_vmseTekQvWltR+r!mB!hD}LvT1vGTt~l1;?j&nVnOa zx+U=OG1>w_5Mp1})hl*KU5`vSJjG`b6S;`2WfH-I^lLD#ETE?Jb8I}G% z2%h8M&jSk?&QPFXC4&eGr@NINVQUAh!~kQK+Sta>Xro!SM>95aKyc_K!%~l6d3YnR zl5qsQn^?(M3PoR3`t60)U~J|JqPdNXrETV!#L>$_e|u>S|37~0J(5*sMfNI$5VALw5t$)bDSJmGdxdO;NLE?Nh(fXonb~_~W{=}*=bFg%BHN9_vsF^z5vvqK?wKijRvAF-h+QI%JuK+)dRG1W}TezV4Q?H0e=mi z4jc0krIg3h5m~9E&4|Dp%;&k3i>+MD%tawMM3^CzL^UrqBR0=7$jJCf;b%pUxXkXK z8!gnokZc<>@QmQWaUmB?tNOs(2OVwV?>jrh#iYrx$xrqWM4X7QOWU|MLH9z^><3R*HD7*5l@603tMPQj#=)r{9%hJ$h*;g& zxcV`OMBv_-RNJ$Qc%IuoQ?9QgUO@?3MH~E05riq^_xLKwxW~h|QgD1~dyYu{x`Kv5n z*OieQdpyyPdTv(S?id=Pi)GXBc^>iXZo_guiAn-5&pQ2fmRGclY~wj+&pU020*T`8 z7jVfKxuuIW0)%0VV^_bxXC)7B=NCw~sVUtL?PfA?UPoSvQ-A;-p$@Z9=j?0Xmp-#W{} zg5CX6qxstl1&h#vtB*-V?v43zFy^7RY}J_-n4C;!ALk7@uvdii~P*?)mY!{dc(J z!%t^wy^u%zuDw)0BPMop!0#x1FjDDecG+@)6J0z5CE?9Cw+ZAU8A+H|bSXx?w;8;= zy@}dfx0dvYnI!3H_>5#NEziS9u({^4#2(;bWMAny&&q1r9KnG2!MBse*bu3g-KGtX z2v#>YAH9G7Ui-}8qon&LacOB0HhqPea-q~YxRaB6oAc&-8S7g^4ljItsKW^Y&7%PRz*#eU(EMNoER7Dw6N_R$28xg-R8~yF0Ur{ z9qH+o7fn^HXV0FE>5l!%X!brmIJh(z25jKn12#qp4{U^1+>L)}iBLl>kdSJ1A;0L$ ziPIc&*$GB6W@cxFZTk^VPfs~>B@K;B5-#`eYx7gM=W)Q==t#jZ*~L=S)~2u@uEIfJ z)MWMa66d@QGS;WG&4($VjJC`x-61O(){vFG`H7S;5GrXRqo5|^})~A$+>4a6CJdTXDgzkJr#8TCrYITj=ao9A{WVe3~+flInF3eBq{|w?% zhGy_uuPAbFzKby%Dnw;t9(U3qK4({2^Mx_VDFNs!>zV&CJ6 zic2Y}sahpwa!yPACyozSeAVjKi+_pn8rOyqoIIc5vOJhmLe|jGu-c`<_pC{oUDe*6 zmzvKgjQ3VmhD;#gz5Dn56B1|^`idgaD_iU{Z)!`MtlM&@g&^|i`(@1Ba~G*G%4mpOU8aS}h-8qn{>`Xsx!kAG&b zC+`}qq?d@y)vM!6OSVap_ooz-Ya(MlSucAv=H24hs(+If+r`*yy9}R&?%C6VcMv8f zrpl_Svx0(@PzBE(LG8ud7`lIqHPz|r`1E{weEIOv;eK>#YPdt~F0S2Rsr2sp^!UVt zKUDoNx=S5P3Y_u%>*5*{S+8(}80fO^BvZa5aaf#G(t-u+bF^z!YSTyRkL70c31}G7i}|3NBb0?O*VJp{rveeSO4w# z^Xe&gsxVt_mW!#s8c)pHbnomPwTdslrAf=tfpNFrviu;Tpi)G!Tn0u;Ei!@kR>+f+ zY|o(%H5x_2LOQRJMvU~=dcSY%h&b4QK0)~H+c%v8->s#AL;-W$`}gl_mfgKUOv{g` ztE*Q!PU51I>Ak;Y+WT7n&PcUrxx*;0SVR!zBKm5}_N2fcKMcix^<1MBwxy+~r@yz* z!*Ou5zp`_arV^4~?{tO!%Gs()z8rz)j-kDA? zUTx*8%P@R*zrVtP36?3}{JPw!ixW&?P?Vv(F>BH+Rlyd5{VVk5typO~mErk0*gdG-mL-A|9K$1I&}!Z)YS+ z?{l=&-fgpSauU=jxQT=0>J*&b+gi3sN=p|YgMQsO@YYJeZPf(20##y>U^9uzJEO{b zXGVPX?a=E&+ccD#z$?cGC;ge31q8mGUwOgb%Xm5K)o|C(01F8=ZTe>0dEeuGCds26 z^X}bp<2oPcppDSwR-2i8$0DyBNkJtkF>563_I&gG2=8(8uD(I|&-j#*7-V4K<;7OC;+gUX|fc>%qbJ-p*Fw?vfSb4oUHHJPE zBg1*I7u{gxwgdi9O0YifQ&mSsM$RQ{B@5eOA}NyIr{QX^W-C9H9UTp!;)+U442E|8 z0+2(_e7R)r=Ut!I)?`Y%q7!R}TxZWVIE7U>4;i<()M@WmQY;t6_&h#iG4w%anb?4R zd0R+(rm}dWK7!$LvD3V^ZGUn7t8I(c$*j{6O3LZmw!7Xrl*^bZSt5A37Rdwam&?sg zDxPq3bo37m4Mn67<~S~0b@UbjdwO~(iXMClK9TclxZ1sY!mCq!Bh0$C!E0yyan3I6 zDfX$UsYJCpk6*98!m5`sz82z)LcyH*bg%U%8GVcGIYs(!<)M;NAo!`5U8#w2uRe6% zTprRzn6dHjh$ zE?C#UXg}~q4qrbUw019vwQDm(@q{79tNIMh)8VTXumXyk32eYPCjwuF-+pT_wC!a zDfcAr_P*wD^Yjd@C9~0gU#G19zHu1uX~z@FGS+Fw*SY4Fj%ul5-%D!&?kRk3Zx>2F z>NEBwC@d_*L^73P&*Lm;e>v13k87`B)~oQgd)f)BfY5oN=fv~p&)a661P7D9(k4Y3-ZsS@*!KR%ZxB8Kh5y4T|Htb@?9v)SI zohdoJ$i?BE2vX_g3TX3hRsUZk?88fmfl;rX!o5r%FHw0IeB2=van4_ zfQXT*VoreZwh7W zHAA~pU7Q=Ot7~*tO_vSd2ER|MN@by042TYOa zIN9W1QBg6c^j;*ZUl&K3yeaIIc~UKavd~4*`%}S)6zxUJuy!+nLN4DzSex6J8{%%O z8B*qUKPnw_EOcJ$nF<8KtPvrPO3a&YrZk^sozGymTc);Rx_(tXE0>2O(p=j`b7{10 zwA4G@drLpm*u-ah*fpomYk#?dNHxKXj=*1R{ZDfPjE|(_g5*?W(7WStJ#q z^;I(JvwNB^h2pUE89AQZ>3+SW$@i3YZlaw5t?@h;Yjgl)b9Zb{gz_cXol{Zv_z}IJ z=XJQ}+`TCsfJ?QThJim^<$N;L0ctoVGCrH`E9`01VIEGu(ySOk8q%42 z9kb%lR>G@x=Iz}oc5V9Ou&lduiZ#48`muC*9p>88dS5mkru=uZ2Vt^@dD+{ybQXoV zI=spoNC|OH5#x@3B)cN`t5Wi4c4~@6eEY|P26W+lYHXx>C&>|t!hsxjm!*EKxy93- z=b3!fIC|HX-gYMSI&RKsg|?2?`Hq&xUb^%3gY6(t8MVb7_#t=UB)kVXx(Vs8_!xFg zhrY;_FH16WH`F3Y%-1a7z7SpC2)U8xa+!>=1`h+Jw6rvJpAew* zZZE*Av4ZL^LTUJtgk(ZL_9tFA3Vi%HL4=fJ>xT5s_El9=CUsZ>@Vi2xMXWct{Y}yq z{rvne>+mmBJ(!J%ii$GXJBH$|*6KXp)o4ciLLo9fOH#Q@T5E8Sr$aI~C`D1rK%22S zYbj%{zr;M6#8R64(d*aT^?iMPJ@uGmQMqF<;lm7hv z753--wVOIRIuGX$_kNx9^72yM5ET`Da#XqeRM+Ul67YfQpdZbr4eFTAiHflk4%GqR zvF$GRwH`e5XSN$MekC06V|MYmqT|8Em{)epzhIlI*=LDGDm^vl=)}9v$o5q7eBO99 z+w}Ica$A}>7j$b=SxFj~lX0!VYov$nKVMMh8GQrZHN)*d=K z)Ij*Hgl~n|WyO0LeALFkLAHa^d2|k_WTu zD7u6Nffcp=&Jk9ZNYnhJCSvh*jlK_|X09snMB|dZ=^`^O&qbhL7%F2rIneHH;o82}e<6z(M?%h{v zMsyK(L2EqsE4 zrC;YGFE&~6(-zwfyus}5d#xYP({ugX_8B~way3y7H9HA)p5~H+p}wRIJ1Dkt2|}F9 z!rIqrL$7}EnqhjRqq;m;rWUD@CT=aftRxEadw==A^XtIU?7j-E)N)NsM1)ezS5Hr` zq}O5DQDMQ{5lXviWQFa3nXp}7;VnfKm8(FpRgDj@j!7@rIUVdc!p=!*H)q){Y3f~3 zP=}>nCc<+(R#$#!Kf%S%HgE7qUb{U1wMDCWX*;)PF-Cfhoxo@-*9)fRRM>~v49)S9$SK}cB3k(AR zjx^U-#Kdb+Hj^VH|)vl?!@unCeF7oEho4a$i@#cBk> zDLN%33}8GeD*cN!gl1lS{M@=b=lLS%^18P6HEz~5=uu|tQ!SM)%QvGpEUm0s$+hOE zuMkM^9^cT)-VM|??MS_BQWBDtG7JrHZ!^;Hx{hAbGHQb!ukDY`!adr_=YyTzGg#(#T^z+pIvB^Zxh3 zr2E=QgBP@i9J!CT%4WQheHlxP1ruE8@;F9&i{&Ik_gLfC`h1alDih5$>PeqwU6#G;iN?fW^_Ko0z0yf|0o zxBSbYHA&SFtrRC2P-URb63h9yFNC}aH7JiK5uGUdmQAKdD1ormfKmwlwF z%Il&jmpN@(9K+(*ceX44y;1kXbq48Gyd~MW>@MU1CQIGUGFxluN1qjnSm07AfQOC!VpgYWt9a)vIhM>0|mXkx_@?Qh&hZj zx{$D?rS9Bex9m&{2m12dYj@o;fjsEZqcQkoLiwBGjn=^fGSIXMYw zcnzhX=7B0*4D;(W_Z7p)$O!RElkl8KF!qrK{S8!uJ5{WoX4;drYCN2~3(6VUW=rHF zw$IV94jtXSvpuUjC>&5WB_^-pe&A_nJ%n3+Q`cJ2oiX-obGZmpT;P8Sua?Stsjxq@ z`J6acGZPa4=tEF5#@DA?5kC}XANSSs(U<9aAVg+OrNH*LVi_eZR(tZawM z3qF(eT`%qL*Xu6Qz{JEPr=`tRq}N`_NBBX0TKe(sq?A#>_Z?HKfj1W9*!2_H6Xn6# zf^c<#AQNK(F@Cld`pnp87ssF31geX25F1h1bXHbbcJHK~V`s(gtbos{&B_@27tNDe zS8I7nEMU6@-aMig_fhFQ=uF7o@Y2uesV(UBmJ1Q~@+p@jn;Cul`iX8)U&6vK?vlom zO5Gx(oS_*g(>S0LqFVUcSUuL;e0*q;;ppc|jU?OV zYd&eW?^=#;o_7)^_*7kT( zk3JUk@0 z0T`_bd~y;y*)vv~190XN+t-c$2-~li)-r!5DdZ-R=p`#N$&dh8oJX6MTgZumI}eyc zDOHVh_))1fB|*<;Sm_6gZK5HNNZ6nmRzawtf{TkVT6Of_TAq=V9NF{&RScN7ET03= zK!xL@!!?l3k|o7c#heLGojMi6!^bMrs7=3XQ>3Y_n?#2UfYNF0u z1;bNG2@p9JDZPka@+F4}V73?D*S#n(=O9r~P^jLS2u0R;FWq`;NrEb9pgmO|thWl& zsVJ)T#K*?;#@~!I->qw?wFg1*k%nSMOHr#n9P-j6y!I<)H5(lqA*kObioC;9pbex8j z>1}LxeM+VN??iE_4Sk4TizfRL*g`s; zHr1mH%#!Rn(_fydMe>kj+@EYBLHXaF7!~JrDbPF-L|*I{aWxjA^W(DFkYjCM8$Fh{WcJy6i?;mh=zZKxSul5^wsVZ)8 zXGdQdfwxSwH_+^LSh>9{ncI{SmsM`-NFUoUIXl!)*w)t3(FA22LExZfc2ZWZ3o&ME++z=ew~{ei~V>mi5{@a^UaFR78`x1>yxVPNX$Eg{+q zdgl4t*9#1rFiz~c}&dt)m+kT)A(HpGi&nNvHm|tAv@(^`8N^5|o5mGq>rA5 zXCd(He!-nqSZel;wE=i!6TrGwS6BHD@5?MW)AAYY0pH`Sa-6A zuEy-^nJ;de7rfaKl?mZu&$8X0)lCwSHN#~8xN#Gkc(Cwk3jvLT10^q%9W50qrS)CV31PH&(B+KfP-zavpO!~b0Bi} z`>Vk15l@Xx3A>T%Q}8P9hqE}+AIPp8pcXF%f~x-A7+1giCTy?2RXN*qPiKPz==STo zE|DRp`^2XwX2TT%$Mrif^f}FR!nOz^2>e#?Oui8m15n^`nTr%^K%{CkQaDnv!NSB; zrizT)zw2{OJ`9QK3rLTSHvL}2V@IW8P3tRt9v>4dn=c37PfD$ANej(nRhC(VP9|Po zPj}c}9(oM8^vtD8eYRrItf7vTJ5tT!7EMky`2dg;(tVQJWe&A0SL!vfR zMm7PhN%+vwk0+uLRLz`D`*-i&;S&1N8{Tw|D>nP~tF`id`OTC%r)?@%xQ zwRc3+^Ih{7;0FAw1|U>DnNlH@K8IDSuz`sXRbMbHDZ#YiLN~A* z5RyOus`|OQMQPSmMMXtmWv|Qhy?0*JssHv(3v_vRcLKiZReux=W&%|7Pft(3H~00- z_%o|Cfr1CX)*4{?0>R4+DO~B+xFS?g1Ph}fSL+2TiV`x2Igx|che)}t)NB)mD}k2) z!UtGiv4|9`ewrXs2F~i;nPi#HGzlCqDKY%Mg5EU_JP#+n#%=96sLduGo!?*GI2lC7 z#9Mv1HPjHwSIdqv_#ErW=13aYXIZTV=3v_Wz7Akg$-VG$u4_JkV6UHSg^N?$IM2e8 zmqZ|c7af(!w;7BWz_aD07zuN|K5 z^9Icd(@_#IoL~{${puI4B77*2)L`;qU%q_V6fQkx*5v45mwLA4*eHAh5M8^VF=#_D z+lG00c{y%I?$cD=Ro)=cb%J-&wD1dXN=In~aMCxQ9H(`Yyl<#FIhC4NS%D%B{cH8>N0~EZ%xo2lu&^&(x|HcKS_?UelaD}!Ap|cw z$NU4BU2+xA=z;y%11@R+tn=v1)gXMh7a=<`U0vNwP;umr+}+&3BmvU?te3-rCwIPI zZ?OCc%99LI@xryWwI4xMP8>0Sn)XP*yb0gDDeS}o)PFEgh``nn+Na+X$?LeN%@lgP zk$#-GydB?`CNU&v4Bjl^t(P$%R#r3F3|*D){rOV4{4m>oo5S}62 z%gM+i*rqqbv_dA+uiIhe_1#E}DZinze@gSq#yzcBEId3s)A5EtIUm0BvyR$%<}rgH zS^1<}FS-Yn~YGL8z8M$Zu@K%tODvgkx!hfW~~+hFjw4ei8jbHva zMA!hZ$MasMQK+Yh>)%PqmN-%x5RSTkE09CYpsX|~uC|TYC$WhxRi8ckoUOOUb6fMhiyb&? z=Dyv8UJ z{&|+0n;12ayT&0_*`mxdTMvv4k%AnNnE-i9xSj9WvyQejxp#)GqG=|n zybTk`E*$MllHInj&`+6tzfy~Z^bZV}0?8D95}+8~zr}X8!1R5_k}1SUqnoZ588JY7 zQ5J^iDe3f+*r!he(NqM;S~;^YqfelYKZjZh-Cdhk1hzEYwRLe?28Q2w$=cf5vXjW) z&!O&+`pVJHIb`fbq@)b&cROml=)vM;4aiivNDnkfzX2bZB_=Ko+t}E6&PfcY0N&Bj zksff!O%I8V6wv?(9kDZDZt`;UL6};?#C2?WnO%L_?^j{9>?j#IpaJ z$Nwjyi`yKMN zNMObXc6kL{#)lAO8ABJ3x_Y^YFK8f|MMe@($Ot4nHc`dSA}s!Qh`2|#*Fb#jBN6&$ z-(t^PzD$Qk4}EsV@qSb}w{CiAG4TO@88|j5w|xTj8qEv}KZ*VGt19uNgg$O(&};R$ zPcL#0f$dcfoJ4kL6YAGJSJ%gH-(Cci5Cyp^NNT|zy;{^qlV}V~2UNKoU^QsMNOT1A z_nqYVwO?l684pC?d#ZzRo+L;}1181PxJwKa37RLPEma8%WUrIs_5E zK3gRLP2ho2M{>#j2ZCIGt#`ur@D~lN9hS+<=p~*#o84a;;Ll|A`{L8+VO@U$D;)s{ zQ@yv4?*`7a7Bp_{qHT!TCW<~dzx8vN2$p1uh@(u*FEkB!BD!ke_5ZvBXl`;~v5)=y z83A(33&097bxlz8R(lN`G6A=XI4@9w?9Adz^84ep$&#HkSN}cK*^o^|X+#9mm5AbE z{^XhtsJjBtc~F3uc)=p1-RSY}D~+?V3Vo{n>;et|6%nxI)gb6;0uDS{>(J~VDyj>+ z|M>{auI}z{*(x+J4DC6oClP+g62HFno*DxKL)e?)&sC&N>1t-Zd^rigULH{1H;CJD z=e z8!3C6p?vLoeNMpb*u654_MpAY^P|`P1sjr=sBsPd&hte=obAy74s?*|Natp z0yI|0?qVR|BEb4=%khQAruW{ZVD3INzb2^1{T+NCD@U#WcT{2r*?j_dphro1@0mbg z56VCgGd|8yOl+*cgBew{#sYSjk5W3z!NKvIK|9OtH4(+fDrSM-Uo@Mbjxpn70;Vdl z8xn?U)+@*Zr2#@5r_hjbCSZ>pND!}$9v{)v*46^O67zxZQnm~m!M(a}>KTK}i8gko zwPgYC&X;K&SSh3R9~vJnVDB(`Q3@D;G$sk*Bec3*cKvArI-Xv&n*iv(5lp@ZCmgER zg1Ss&u|ep~al+4%`RBy}Py?>;7~OM_sJ85ecMbw33L*+rdD8wG&r42X3`hpxCIGf1 z03iJ!*2GROs`PL__0OXDvx0>3!Y3FgA>NE;#3SfM{c%Z7C#vOxwZM5j*AIrk47xO0 zdLYma8IRvHNhJ!(;UK6<84oZKVf!CA|kp9Bts2rMQZ^3rpJ(p%>Kyy)bv&qZ7xXnUl+aMv3O5Z$0YVo09D0Z!Zg!UwKD(_0n2Cg*ih1>Xm4E-RY0&S*O=gdj8z-tlo6d5yth|~{0Z!$48TMe6jZe*d|vXjRk-l? z4K{e$;5|^gYu=5)01>-2FtR6s2i<{m2kLKmY~FGD`Qub{bac*cJj8?1={s}g3^{h@ z2jhG9_<{NP0Tt`NU>KwC}JNp?t@ zQ3F&28eJqr-X`IyVJzReUDo5I`)e)DVM~)wd6Yn@8wV3x3S#*X`#>+*{7W1Mu+mHI z4}LA&Fs$=ofRY7a{ZH34B+z6;3)SqdNOEQ&X<^}_!Uiq>$9q>YKrnBb_OrB~>$r>e z{%=J)q8Zv*V76h6ICT+Mx6rU2GzHja4e0&bPCWWW`vi-RG4ANjRofP7aN+Adu&x3=ilD{6}@N>B=I2OtYC| z6&!DA6yV6U{cEYe&~w!XLCO)Lgs83tM}<&43nENt-O-*(UX*H>u9~332tD8mDX1?Y zWnSQsfgBhLReBNrN5Ril6cbd}#Y#>!qiH7sqQbbQelWUxAP51`!WHX++d*dcw6&>LuQk!ZXh7Ik7L2I2oIWUYph+?!=)Q%p8HY~TT`-#N zx*VIH^>db<_)kx;=meHXgoii4w{)Js@&9)B=Arz7(YR?E{4)(SnD-l>j7ol z5sof_zC{N+2_7zTdjWF6HVd>ITwz#& z0G7Gv3GxX30sj(!JV9!60$_!rni`;2KBpsqEG-nee@-C;!k|n-qD9)!klxVHkerh8 z5d@`CLLV5BYRZC-2?AFHu%n5I#!!H^3eXC_K@~xz} zV9|KSqYy-J=^L#lndSMnH?5D zR#@H`2zqGe^5s^VDna`nvGo9zDYL z+We+~5~Ha2_#7vZ^S~<5qe74rCV@8wE@%RQH++C!_wQdo0ShW-fIWmBVu0%NAT%8h zG7De9pG6ZLrlzKeg4PsJiHgmeNxhDCXVC*H#c(>Y6f}>oi72pyxjiQ#!2k{cCV;|F z;1{U;O3QD8plT=Nzh1W;R7L-dTW(gP2Qr{1!s(~*J|j;O7(qEuS)!AYen)wrxHQ2} z*Pw+?fKsRM?*4RJJQHX{9$QOKDn@Bo^v-Uum2NY01`y)DNq#>=3LaWcOPnMp$Z#A1EKJNGM#<#0icz~WK?IFN+9?IKEzpBUZh*bkP^Jb8A9eFU z$Frq!$&-_ld&Y=|jjfl`+1Q8!X(d!00<@=G8qxgnN^RQ2=PzFxpamBsRlwM?`lKE( zGKr_)o7{>L`FCXrya0w}0D9sDd8Jbj0vqQ!s7!EBhv&_PDFh7A$X2^KF+TnW@Q~$f zZo^(c%N=@1PxQge7=U>+l+rc>Ss^0yR1)(_d~xLQ0-Tu2(WUgnDxL=g+p@FFtYx0#S>6e_b~*0ZF=2d`2+y^hC^J;z?Vx(N`fJ@tLD42dIs>e z>7K8lVah*}^ZT=$Hr#83o}ZfW9q%ov9dU?szHxlHQ z7v;WN*xJ6X;7w9Ht@_mHn!oKOYX)g5Z?Hr2vhz)=2;b zG^a|i$zZUGdx`$`Yn92`v^w4mhwkSKU0tGcYgMIAq#o70;rmr3yhNArB#aWYQv%le zrVsbGE%H=sI;CMyQFuD#tluL-c(cN~`#fmBsFnr!&+|U}%a?xZV3OWqz{7RsjA0>K zJOL6Jf%-S8kd1u|6cgif{xkFlRw7v0qKF^R<>z)oJ z2$Uw?K0OF{Nb~)ibxImI@Gcx)O1&5#HZ2tN(G~WVPcayUAn$#TYX@kGWu=U6(bj!t zyWp`p_V88Hm9HVN+Fn4bLs4X?_xtnG^ASJs?4AUf}JLxxpAZ5=J!nz-5WAz*!>IqEG(ZUszEio z7|sE`h|ty5HFN{~^%JnIp}+|(8iMk~gN`Bs8YLlGbwd~RicK)kBlgG#t90)m$V7B5 zY~vu{q(RjnJ~%iCIrwVH@|`CwtnQJMi%a}%@6Fp#PXDcS7<>;bX{V&64F=IdvFazA zG_=9r&p=4Nq%p6lyaawXe_1x5cN`3L&><@8;p^EG?SZu1eD`O z;aahNTjCJV&doq)LB*UAh;=x@ffqr=Eg1S@z}g4Up6=bdCk1(QTdJTXaR~{v$cq;* z5-n%&kNiAye8FsyUG-=_vgoAzLUtc2TluvieVQ{GgP!t)Z)Y&2Z4Wh>&Yt3>A@pxZCTnC=7aihEH?WB;0*U@&&G2>ul z>z6MVhKY99wjGwmki_9@H=iu$Uc!$#IB_aFJA18{{yOq+Oe%8T(vIAb8r6T8ssdJ)jCJrPmq&q&Q*nVGf8^gaq-Og-8_s+To{nHrf(di zqRolWS1ypmkWxz2k~JrC4VkyWhdsN0eXyijq$#DUIq|*wM$UkUwswR7-hn{^y}3HW zzh08nL;Ju_(46Uh*t>>K>E9JpcG^V0LeivCL0QXwOxFb|IG#YsPT* zX=37Mfro-qfaa=Q-^sJEvO13TlK&hUL_5OX{d7eVTVY%|AGiq-w^e3Pu*asR9-&O6 zQ*xghQp>i7T`}w(w3Hxl3g4mt5Sl&-vH?}914yRma}N*`G-M7M)4(QB1asiNP$54+ zvdzSbZ6x7(h5aocWT!GSGc9Y9dpVbDdI{e*c2DZc_Ydha@<&&=ojqqB>3&2uBl|Cn zwfq&)NP4drO6Uw6YEZ|$6drZ~PzXd)gG?-|BExqSmtB@17pG+psKgCs+!0&|lrIA} zvU})`ayV4zL?8zc0gzkI&=_XI5i&E#OgaM^TLW(a>o-0|VH$N`iR+3{x8D$|g#kn@ z@VTNKCeS=t^1n*$uR3rI@RGx&qm98@ok*~2%?4=#eDz#UwFvUNR7ef{a?_WQK1Flhx5HVySr40R=`Stj8PyaYZHyn zi^@Pap(VzxtgIYMbsdSGWBAM_Au;R}DS7tXznA0ht2}q{;z@|^Hb4|FDkX*6BebvK zC~h4-$sBYEv;=@YjSpqy+wSiw6~3KP&T_nY(5h5ITK1Q4a;}w%^5zDLmqD#-2K*@i zSBK`wp%3QFLzY$+G(a!}P(?1Z@ZK~bM)<*I z0H``p?(hXSEP#>9<%zg8*}tm@>TkrAnABs^Uv`1#xzN*x{R>3v9LhKVg3hs+-h{l@ zctOR840s;JaIWqf6UiwYJiKH^8ljBmj$glkkeXa`oRPr{S{MXk^=cxTjSc<=!HOf& z(U2np;o$|Sb9cJ3p7iANnE%h zMqp8axZ41tKqM-{f_;NhN{~~3Kc{B2jtO!w{eYmXZWf--gWF6_LE(>vsQZ2&R?2CP zEK&&niyOeOjKf2X;D$_~hNH3}JR~C9haeU-IrG)A`W__1H80EWE7sJfQ8DvZs>_ZRbTl3iBqSZBqq|Lp=6NBXl`|as)|f`ms8M%s@wfF z6#ch}L*Qyq6${dr^^iEUTcbdwv>+0?ASh;Wa{-#BAj7_{()|g(8PTmscyI-JpQ!2v zQ8RQQ?QYBp*bNHii?M~BzGNQ;n(jE{tAGf6f=URBE9k=u zYs|AsqSf!qrif;_!hZ01D4!(H!(8;wYT63CfXY2tSuEKf*?XWV3Ba?c&{G=d%_u4c z&w0&urYis)e>e+C;j8fI1R8MC8zKDj6;z0~^RSwLo8llK>bEYHYs#b22rLOyPDGDb zUJWAV;O_(myE^yn9LgEs;S7zwF9^r}ucT`JR{9M9!jR2PCwPVnDJr-;mka$`?fw7| z9S$rid8p-Dki~I>C;AvF$jD%#=TYFJ$01My4@S8RwCxN&9QMI11``x;)Ub}jL)+K6 zo`*5G1RjZdtW)p1v07vuqL}k8j9@;Rs;^?sEAh?@gB4_cu(Jkg9Z#t@pCz2~gwuo0 znY!oWdcnE;3gP*=Umjj?s<09cRyR5_b00o0)<~C}>wZZFl;zQ-5jV(g(ECBa!Fmq@ zCn0doyP>%m56W!zoESKpGo9&?&_+#w*yz=Ii9#^cKFJ5@>$R7kP~9HjSHdPwYslZC zlNdXK9zFv7>my{iLq302ge6@4fhQewLkKaD!bvAMHOLG`FLW%Fn%fud7G;EQ=PdW+ zokX8<1JAb@y5RLj0Gw+u@6dz_@es`yRPBOfIUhQapf^0KtE>CoDi5k6I1s|2JYr&} zQ7RxWkBd-NddDULj(Q?_4AngogbCEFMW0fF2CU&xQE+k`51Ljm$SI~UIe|xK;n-k2 z`Un{TvrqV8r-caN*+KG98XVz%xgp-Py)xPiS!jFSl+ldmQL)2DYncQ1{{}g)b9X(2Ki~gTmhodL5eLK!*T|h%{6xD6f{iy}l{g zT&J=qz~n706;$e-9v^>*)?7HCp$AX_awqN)AGkk>2I_5U&KYp&dEP}1fc`-7L}?0H~pieR3hRl zC51o|=rJ^iOmGPA^cQCuX5K}u1#(bspqxXOm+NxwI~X+*zcGT9)=0EcH|HDxF=n$l&-R)}lF)X!@N4<{@yRF}yL5zXFKt)Xigmqemr#R3Y|IvH{ zFMukrNHIJ@fS`09oV{xk!HVjCtv0xZm?!FAKD-_t*J;1}Kv%@bPK)Yk%`H26s;5G* zia_KLW?4pb(r4bRR;P7m?H;>;Vzz_a9Aaa&7 z6rQd{j2qBp8ujHrJu`UfFZ(h`S2OB%uW>L(@4;e6^H{8XnJp#B(;MLDBW`@M7n*-x ziics_1w8!YV*`TJ!l^|9`e-gSDGl;~dNf=I-sVTh7|awUfG!P5HFy}$;6YjLrrZBI zzgM3a5(Uv`9bG(`H#>{S@O~kyB*5z=&yR2SxraU9M}20$e)Q|hQ5M)V^jUBOua5x^5bDu7zLio3(S}6(wvg%0*V^q!L)k|hHzKlKg<;(@y`@e(6DfRVZ z51;9D5rm3=ugAhPw8@9 z&nlLR&XBMH|xHm>~#tdX}g4-A}w_*(-QpKB0; zHUl~Po|9ADfipZ|=P`)bXCbZ$8QZ+6MYf@_HNF2ysl2xe4Vro}S>Su__5;Y}vUJz^ zn3T-G^A+4S(BuY{vX#Sbk>CaeG6qKy(K8akkj{m6Pc7m&@hEE%Y^PE1OU+R(3{Ldy zhXFLoWzqkl&iLxWm5bx2#VDuuje=engQJ@M|FQKQ;9R!t|6eOAlp>WGiWExO5lL30 zMI}*EDY7>a6=f?edlU^Rd(UK*k|f!q5Hhlf|L5v?-s<@s|L1s*4sVZ#`+MK_bzSHA zS*N{~D5a<-oo@$+_NFZ@+17f}dV@tK#dlLDuN|f_k^9uQCFAgO#{K(`uH@!s_L$5Uu=%wKM3>VARHPqa-&)N{QHq-6 ziI6Y9ClrGlw3r-Xu-d(YZG${iLMG+d_IZK`J(Z*h^$+Sn_}4gyL<3E!;&NJI4@5tZ zrapI?- ztB=l{5OEh8$ujX&Xc{}x(|*RPV6N^;b6QD*>ASAl;H~duvWj!ROjhJ%=I(wGJVth| z`>p94Y-NqUKC`Udv~gh2c528*g6xWPv2iIyZEc2q%hj&_-dHnwdV77)JQ`O7ts`V0 zI2=X_%ycCv@{&#j^3FBsqN$gIEx;Q`tR}ZuBllmJ+-&8|Fh?qxWq}(fX4VgeQz&u` zOP<>-_E`6W%9_U@UD|NobmGL-O#V7icmL)qsl(?LmQ8#=tij2lzAv4N73pQ9ppR+j zuZ1Uw@MnEN6&iIpA6{;H^W;?R3;iPw3X9a&WDi@}6?rVWyt+<&^#P%KU8cdAq%^f{LsVA(@Ht z-AO*Jt2`_)d>-qqD}SY#SfSK?K3{*XTUKsXagdHscJWac3Q6~%r zyHI25JXqqMJrV!+D|7R6R)Y(>=Q@r@Jqk+iFBcD=+s;)JnzO5?{aSd|K{c75R9C`T zrKdhz9!kBiB>xou^r!yiV~xWX(-~wVBK(FtpRRMEF9VoHMD>th!{_qh@!{&HUY+EH z2R?S#C>tF!QObr%*$jsW0wnw5QW0$LTA%eQx+W)jq_zleo z<{MzN2>yd=i+D+i!#%ZcDF2S$IoSx3MCthT-dVzdk=2MSRY7P1uJ%N_Xn%kHi1z%v zh1mJ2+|s%-s=h{Nv$ZYFX3mO=9=n2r@4xt;?3Aca)h_*g^dI0*S{`ISvNS$&fz?xi z>*7BD%LgCSylc!|H(6^EZ#Ye5dZ&lVlahJAD>^k*>^ZZtE_KMhy;t&88Fp~I)oC{f zk~lpS4I2$RS{SmG!MSi(OAfs%SyRyEqkQI6a9s_jG=;MHSQ-ObzEWV#HOx1}`S=vL z8h9Qe+=erZ=qi8xlPFYK-nMI|euyeUelD_!xkn>nA!7B|j^@af_`KG7BDspnU z3b>>c&z_w>>@Zrnam#514~w3{CHYmJx7N>UWfF4%I(*jNaEb@;#CzOGMXLTF1C2@k z#99C%gzq`pDbVEW=ePUx>G=QDA^)EnbaQ~dd=yGh9Q>xOG9wEE;dvb`)(RBfP0pJR zh`FATl;3n_r})Lqmo+PvXC3OhI2--w<>^AtPMqw>$W87|1KdbWsS4fX4b9?Rtvcx5 z5ej9>E!=eKwoFm|)RazT!%AsBy%0w=c@)N>%$E=O{Kk(2cK%gXNj*}3^@i2_->m^o)+Kld2#4RKK;sj5{Kg zhraH1@BkMpoDc|J-Z(smm)r``YQ&y_7KfRcPI)NlBM<8E@u5e2`cF?Q%epWuxVzTMD_olB($ni5 zFj;*4!@3O{7EzD}a&98ec>4Cj$3+)**?@ULNc_5WH(n`yEBRG?@=gC;>#J*x8cM|M zyY3NZA@BR++rfGF0ah3GPublDbTKxTEL@o-X4z@y>bmr=x2n|a%OIBtH>D+HVPpLb z%b?QS3JEB|n?UiB$Y-2Q&r>%!1|}0J5YY`vn%!+)caw=0^~RKnE$ZG>u~?a!D;YaZ zEZZ|L&s@#9xbo348L8jho@mxv$nrKVXWQs+r?eh8MhMSer&!pBm)RO(@rG|WD02Mg zC5}IyV2KdhbcToVsFSg(w4O7wtek4&DS^Ixmet*vDt7&aEV)>{=-)i_AKpt6QY4uQA5b*g#g zem;?$>Z-4JrABtWxNS9@v@GhQuNmwA-nifyhMCO*-+5@=j%}~9Yg#5JC`QqllUt#z zy;j`9V#`m%sI{if;dUMn6yy~3pxgDqF zur46R*N(7+ES6+c5%mPJR!Q1j%=*^Peng-Ha}s7XA}>l`yz9T2wj$_ zYFeK(&{2Qk_J5y}K=&D5D~p|{TbigSY6@Ff6wKK-aUSMY{;Iq~b}Xegt^4Lmtw8y; z+A9Z_Kew5+6i8o^{tkVKI@0v zb^R=Ibzkm-hYOSc(Ig8vF$pV_$qM;Q`RehRhZS_KJu{GZ(JH5v*XUJE|GPd9Q-kxc z#H+vRRojSb63Gy&iiGyj2u74c3s)wm%ap$FBiHx6*{v&mM z5{9~bv79VQFSgpgiOBl>&zeqr%4LsGg#l#G1oM)vr2D{y1AsSyUs$+CGTR8{h+6|X zj|3AEk27%`SrklfAb=t1aiDLt_SV|Hd&Q`o-R`~Cx9fk*W>sCv;e*cHr|KEeNVT8ftroD(*XG6!f;HKY)a0frV#!bgAlh-)>YYBi z&yFAMP*Xe4JOnqixQLgV<7lJKz*RS#fe`t<8C=&a_x2CYDTH z=U14M0_iY%0~5(N_@IQKMfGHI{id0iloWe*_S~sn^>`_U^-~9B&xh0jIC!3!`rV$J z*=<2eryiYu%4r!9Po(Wvu3RA;X4$v! zXhE94uXUz-L!08<#M#Xb+P5xvS{qEVJ+YxSdZzA%Iulke^|2FYvF3TUw`$iVw=^Y0B&vfNT=De(R=_I$Eio*CGjQ|I)}Mnb2t>F~8gHPiJ;C?bEDbFF1z zTPG+;56()|@@ksIKX_r~6N4)8jQ&5W8oNrZoEuYITqi4hSR%gnwozYtYwjJ9y6;89 zo8vb>qhLUSzaKDZHpfd)KUGv#Ue-=1{Z-xBSy~F!dUNxI=$$O|xHQL}^vY?_q%hL= zmSp|~tzO%r*9IN2g|-=)bsdFo>xh8 zLO+YW^a^O`lh8Vy`C)SAOec(L4;PR&O#u&jTkd#DFrp19?ebqL{T}30I61zC&pC24 z%Sd0={1zsUDUZzGm3fRG7C@NYE=VZhTl|%7{gjvCs=ZOr%-=mueOYV1IIhn(t-7r! zwXf%0qxvM9D4yuA%8lH9#EFR6M#4c0smnb+zHErUHvSEn{L+XR=Lvs)ztC4%_7`lT z%i|LGc=-H$Xm`KKGj(&;{%GDMOPBsZrvFtI3Y`VSMGR@a`OSnzB`Ws+FSYZ_^^aSs z@us?ZDa0J>kCClGZIk%LXS;OAqJ}S~(#p!UmfTkxVZH%!fK%R?m5g-3dq*T;(C5)q zaBO*ttscU~gVGu)S0U9x8-GVE83i`Z^S^wx*Pva8GE6hg3`G390ymfI{CqNh-?lHo zmyBh&f66ae&gfCY-D;sS;m zJ7f3Dzw}q;E`1H|C2^(vKY(qVZa#7YIJ^l@rZ4gKb}zZn-F=#bszTXM86KiI!;)yG zfF+7dm3H_Ni$MfD$E+-zSQT8NjcTZ8|BG9kBHi^&o_k=axazKWuD{{zj8OY8E&@DJ zI_=YA{Y$A_1RW)!^x9qa>z^c#J=>FrLP(e=NP2jqFTdM+B)U^I?~cr3XgZ*iH%sf2N0}De>m>}{>J2J z)ZStwkPp1L5we~f`;#)!>@!fFr*0b(dy$%2N@PL6)tI45C)voe zv8L2m<*izXoXuB8D50(pcvr}BA285iY;Q);1QDmk3uuvH(K&KiOmU@R(*X}oLf(wRnN(P;W))c>l>++iNhX>SN#^5lSKQA3+lj@BzeQH`;|*;Ftoue zM}5ne^4Sr)btGm7#6NtvO7>Hn)iL;hNZnGnFwcQ^>E@=rDO7*!c>Ay~-Kyi^9qjDs z+}1nL-tv_lFzPxBqlLhKwr@TFHbg>Fn*x6iX@Fj`1Rrp8m&YoNrna59ogjb7PUYl} z_ba{e>&HnMzWv;CL_1L^SaQ3;6q9NZaqCnN)k0>aNVL^R-h_DB2ptCz1Cb*JnoR0U zmj@|c<;H(R*=rG3G5u0bt#CAHic!{ODs7hS*~zLFu515Z8H5+ecvyc-a@)3>JjWw> z|LX{T*F~+`ee9|4T@&U7`vbiL0CZmAEsEzRf@nxp_9JEFDW>=UL|he-46hV&`1y%W ziG;xryED;a68jqT3U_4Ahb##!|6Suc^ey(Js#GoK|2ea3WQ{HJu+y3K$CVdVaC|bp zOKRN0!DVo>phwV58^ETrA68N9Tn}EBlXNzOJ*b)dW=wW}BmU-3O-*F(&MG-U?uLh2 z9A-$!gIkX%iv0Nn^gI7z14X%tqsu|Vy=ay>*+bhF(9bp$PQfqb!RkBrIc+16#Neoi z1C^&MmWcpA5CNeyr*|r$h9lL;9|Sa>8#VwXXia_P+wIUzko`4ao8GGt$EUyT(09U( zD!`1gRQSy)yDoBJpR#La=&3UJ!t|Q@SLd`z2#Q7qu);tajUlkn6-gyQBrqGo<@d5n zFK7mzth6<9(pT^wR*xhm@QphTw^QM0Js4}!4>L5*MY6B|=lA+Wzuoixatv;WE%U|U z{<(zn*i7*&RVOwHk@AJ;^5HREMw~e~t=2*}wg;33g+jtjP}_|F%l}0L!bk+d2H_0H zkjR3otmYDLBs0t?MdjtHwbE(^e?18@!D?)5Y=uZza4;d?{_j<>M@7Hg)j6*O6h_51 zr>1u0rHBR3gI6=eRduERZ6pEA0~seSWfZ5Sh)^Ok59-FSc@SaBnQAOIG!G{sMz;3W zlpMoVQg-l$$o$sx1v?YJ67)h685TyTX8(-M<2@Z*nOx2)U)6pv-?+Q+u5 z^ z`M%$zhqmbW%^9E|a338A-{G_ZVl1MZY47SX{huQ=)`CJw3^>`?9#0 zikUgZ@TIcwKhgtI02~N}(Gm0!QL8!r9B+V2zGYVrF!)!XO-V8}jNv4AUR*^Zqv)^0 z*Fao*{!RO2vt%!4GZn>N^vnKd{R+}EJCiI=-`EAy{G=nf49IL(c_H`A{CCJW-ZA%-nO^1v0cA>dB_5(Pr=7z#~tU+yB~hheB+YoMURMaqp~cOD@W+J0m^wVZUtr zE^>X3zlkD7)u{Z+HK2#o?Nq_9e^X2qE9l#1QZ_4py)@G&2p*7R!vVg5{I?W(%-uZ_ z_S!I7aznRwYpX^I-QU0Pz16cR#;VC&>WGNA_c5&3Nhxxep3YR$((?KHbwNx1`W=95 zZ!s&~gc;)naDFo7iNGH)ARg|`Te)Fx5f)pOKI0Cmza-`Jxs<7|O!M zWj^DMR~IR1R;SU^WJ@MudPupn zGOa*22m>kzL{LdA;_fuOHCJ@}wHdBT*cYhuv1QQm$ySX%9f4A;0K9v($%Z?@<<^cZ zKmkfT=X76v0|U$P3r2`i7~6X}z@2W!n`WCaz>DIjFDCvw%7u3;a!V-`1h1s_bj_b% z$}p3+&^yr4*7dwO##F<|h_}_Lhl?0%G`x3-|E;eSDPqe9+k9oa@13|^8GeGDXgIWV$9`d?2M5GKxv<(~f{O6Pq z_zJ`*%VqN79>&aXrEE9$uGQ$4RkZ;A(6IIFTc+-{ zw9Rw-1g}hGv*o;gztUmL`FfcrAKgz@ZA)YqQs{Ls(^w^Gm=l=aVKpVVVU0tGc+fp# zkH9DVjE9{|-po^1aV(iHy53C+BGh-7XcY*5PWro=>vN>WfeJ7hB3)9w$q7X+M{V#& zB=k>H;hQ1|AOgFqjx3`v-CqQtLu7OO6Q_ukYfkB{W>724pg>cz>+D%E7qbJwat&wW zcusa#*+}!;+}z(!UG&Xna=2vZY_t@EWlCnLua>a!-fE*y{kHh`eue>Ocw>&OfB7Z8 z^ZifO4@tcnUHsw|>C2$5d544j1NOrs)#-o{NaPIOYBSa~m2ck5$Er8|K8Hw@(|_n3 zsi2#VEI==dgQgX2bBq-^rtIdQ>~+wlD55o?6hRUAF!?3~LhcTN+Zo1$_Y0PVPRg>% z>^makP}{TeeQJyp!?3M=QEleuRlOcr1>ZC|nZJ4MRb4hV{e@OEzk#ZT*+j0;BzVAB z2L%e5+_3&w1`14{fPmZ5dgifQ-h$eFYFyg8YGQrsu3vOVt zmK5-Z{FNS%)FUT2zqvEh;rS<-jzceK2=Oc@IMT!rx*kltBuW>L2Hgd(d$}cB)2!AX zzm!a~R!_7sd^N9!TYM=1urL5dP|9vqh1+oEB zKW}dNQ3G(}9-?b%N590MUh_X1xFmy`hK6P3N@M5ns5^JqG1AsIzvku3-7pKQt>D?{ z^&1EB)%_8{H7m8k-oZPFzjc+>v-c|txIQZO=uDLN`ao+~5|{aWei!44ZPxAm-+boB zX|`q@-tu@VOJm-7=U8&`RG`j`6vOU?MVH~(JX^6<-rlZzSyw18s8gJ7&(Wk1Q9H2M z6z6OV3uuQ>8YQt0Qyi*Tlhdaul;Z{ucFG)REc>5q@p%&J=uR*KbQD;dip@S_vmXY} z!R``)Sv$oL=ZdJi#xwp|a>rgf=)S%r@V#&GIw>vN98F92sWd~Xi2R$cgVil&0xp1r z;dCx7)XRS*aJIwRO#16U{^dsz7mS=6?uQgS{1CBnk(bAa&Gvh=`B&c}M2(KV9B3k-6NW9UN=me8tc#BBGj~XjZ-@a3`MMI4I09=18^l z)JN}q{VI|-d~TiLJo^y)@;{B}zZqZ8lK}m8BC$BsHbP?|B|zg(x{~k5(Fi*S!4(q| zRAcvI>kM{zmn5`U_YQEH1nhPG|4B-H(;`?8e6n1(sEs@4dR*p;;Lr<5;G3{l>RCH~ zcWkf3_QXS@R!ZWW%_Gs)+bBg9XOtcsV-F3-r^0@%in}8Hh7;X1rg}q2jdi9|H5nj!U@=f3izr{E+%Zl6|oH z@h40>Sua`GL;+eO)#S8e(2l0hquz+dX?C^7hU7T`gF{DA zitFq^swB>nE=bLHqa00g+w{2TH!7u~IbDNB`*n{zK)xy*viI&@m7jR z06`z8N0UPoTqZW}3E;dAU1asd z{lGBb1EeT`uZk3;=P=WmbdFd@01%l-OroZ3?&b%7a^7k+3bt98D|TL9r&wS`KWgLl zO~Ntr7$xF+P|SlO_JFc&Z|IM?=FOcyMAGMy**ZQ>?$AF|EwauB%i$m8lS-+#}Dq-UPlJ6TX^}Ma3iauKa*AU7-o1Nw%3q7P*05lVU}M3uyLZ@d=FUgr z8UhY^3Xz8-!UmS7Lh$~l76CrFezY=h#&D)BMDK`U4H9-lZtkL@Wf%xg?)^;R4P^*RQ>^c zo!bVo1O*(_n})QA-q){YI;=c>l8@XgQj z3i5rossH5J3YAlPnD->f3ZAq*EpoU;=S^uT4MJbuzjhn)N#sSw0W?z^xfVV8Cq&G89^#8IClfW(nKTgNp9ho?Y_7u~_bo;#0xBhi$m6=TklZ&5}O;RXtA4Zj? zb{x9OAJU2f*OHfyTe0=-(g|JW`2nMZQt+-Bj9S*$T=j-O--%7?iHtby>>A9)nsN}E z3lt7$*|z^|0o`I-fThv_DQl{9giquS#9sK^xN$(1@xYq@VH_%rr?#vnPy`152y@-2 zC%HffLRf{+2#Dt{!z~Di+Y|NuD%su%&@Fpuf)-&w^vv4n1j8I#_4`HUK7Y!f+oOXg zW|^iYWoZJPhw#(l#}-lX{IsJ}z9jEIw5$#BFJRq1)JQx%cIg)@tIuVgqQAKBZhF6L zO^e&1Ue^~Jk3KBFe1fk0$&uN&^5Jr>P+M8R~p{DkK+9}^QLCk%-MaRq8N3R%}P5=vaCHk@#>c?hH9I` zRb>ky=`(lpBbXQ5o-KN4_h_TTr&TR`)ymy74ZY&>x$wOi(rqYemznRHTe6oYZEkyH zGM7hEa)!{@_2bolPR5-d{?Q8S4;l>y z#@D$+P-12xJvR{YCmISAHR}M5RRuus%?F)9aj{e8Ws>)$t9#pY!Pnnk<=C;cfJ`YA z5=w_Xi3Gk=NXdZs9&+SCk%!?X#C-)^9N}j~ro*k^>Vu<%m;&*b5dlKDNW9PR)sqK{ zGZr3o64rx3H6*=?Sc?7p7|F)EWeXy4xXF{nRT8`y+Yu2Akl<_v1`qIHNbUi&f~sm6 z5*NV%LjPU#_U$tK7@8mlIzM~|D70l1P;nlpUrZGnQLB}}c8=GHHmKo6H-z~>9;qlG zziDt|!EoNqh(Wk7ymwR-pliz7+5x6vt>}#4^FvrS9fjy!kR3@TUYK%hSik;WhABMl zAj@=;j!B^0SqCNzyU#=ODL6}sT0V{G;DG~{3-fM=l8&+Awqp)uDbNBPj1I+Gzn@_W zNQk6ABVG?+3_Y&w3N9OIMl9Ui7e7Bl`WOkbAs+)V3U!hqn1qNp6da2PMxw%}hSNMO z;}MLWeVVm!W5R}n(JY4!9g4AYB08ciPe{xU_zGBi85S>QNvbCPI2oCvDgQSZ2!?Pz zc>G~o^hZ3!y>GuVf;4~qQh(gPgoFgsf0Kfc$_|;u*1@L021EvJ#*@^to1QAT_(;^h z>BEsbP_4qlv)!D;4o!}ilMo=}5j8jXH$6|YyXS5w_E;1NO7P*+r{-)nGt7J_%G1`c z5^qr4o>sAT@~saMGxWf%<$liC#Ka!-^Q0n0I*_EqN2Cu?gNx2;lyu7zMS3Z5NItFx zt%Ly)M-A>=3gn+r#_k0(>89YW76qT7KmUA;{C0SCxf-42KNX=Oq-*Z>BxK|AYc*2EO&-tk@@VDEd7^c%{J@#BhKPO$J+r{EkH{q9XXgKIWm?M zSuu`ZLiJcLsj6^DTk>MbFSXTPtrexfh2%8~C2wJVR&DRz?O`OA$a$c<`%v$NFHy*^ zKJh-t#CXf<*9ErGf*gUVpNrk}^{ZwIL>PVv^ewZt4TJ=2;tDX!IGz(Ye}V4}Tua3$1Z_2BH!7r)Joq)!szMxr^ zx70sg{%l+VvklvY{D3d|B@0gdqI!bk8y}PSEvljqD=mnDeffOhi}+O4b{|i{Rl>Rd zVpO*tFV)YuS9Kr0>$^Ve+_yT~|c z22T~LJ6(~3pY?Yf{XEoL`I02|CW)BHw0RmSj=qozx@1m8QS51FE)Q84{h@9xuEA!n7i`(`m?9%vt5AtF{-+76?`{f zw@71R^N;ChrXhbv8Fp|edABE2r=^YSyY@uZC@)*vP_~a5z0{{Q)+0_)i|#hZ-0yC> zd}G6`NaVBQ*`5H@SfM_n9Lv1uk5t}H%p?WBd#k-&-|)6c;=q;@7gcZZ{0539BAvdj zsqOLd^xUjncLP=WeQx9(zC@IdBL@|w;`QdKP33c=``#8Fw}OB5PEJ?xu|MvVs`CYO zUfh8JdDr|nH0Rw!rI;4y=9DJ)UzDDfI=Dr)N$pKlSw(2i^Y5;HWlkkk*(bXa&m{8n z9yU4#!h}*(T3UK$D|Jz6npxgl`O#X_l~O-l^xv5pL<+xC+Mf_BcyS?fTZVy?hsYgc zt8dyX&I}d=d@z&W5y7RC6wcA=lCkzxCbQD>sIaot^Y=y>1NN*psw+FfnH}np-eVjT z6iGuFwzu<8o#2;NjoNKqYOweFyM3MxTI=C0Qo#WPuit@K6Lc5;I6%}mSvFSwBd`h` z8UK<$pUC^aI`gVHxkYtu*VA7+TAbq@iu^RaW+;@YyQMmN3(n@)>PKw9A?({JcP)y8 zeB(>`gVs!|7MsdcLeET3MZ*_u<7mA|KEQ9Ea@%@At97qNS09eJFlAgnoY2v;;gY7h zzQf+tSEsU-rVh9^pS&RbxhcjW|3A6ZK;juxXq4@e{-TyI%8sTZ_#A> zOX)2FK~kbopZ|148!t?+Vw6#n9i|qyNMeh4W@G8KZ?-KmvF3V91J9}9(K@x)6)&XU zcAqU7F6_Bb8WUv2T6OGMU!@@Kr=lXICX%f%FG1)pc+!vOHMqIgk%6;Mk{ab>f4Xqq zmfL z$aGcsGf(?H@4ncYcO7Y-g*fszr+IY(6n$8Z_=1i9H$bU&5J$Z6=K#pQh1`b4{}}8J z=#GjI&A*}h>6R8zS%aQlrox_#(v8s%UI#bs*%QER+z?)#oDe5cbFi$5j$@C6avd+v zlaEJrt+hU{n@!(=Lm1sP36Q|qW?V1+uRQ~LX_7t9p{i!R5j7JE+=Tz|x*$JDR}P=c zY9ecyf!gO@ZysuL5>7|*r|z6^UJM#RTvgv1VG*g)BB(q`rbv(jkHw3cs{|NCb4&3+ zHSYWDEHvNfH8F&!7y;D>zuo=v2iJSBxw{F@<16y=A`aTpvoXgTRq^ zXqTMe=W8<^A6`!Ag1s=|Z-^I3F;uEWe(ZMB_q0$W6l2f(H6#I;^KKjS40i@3YNHU|9lwEahG@g7h=Pbrqagz;YMlUq`eHSm!lxop zp(q&;&bRJt)_VK)BC6LASmNnQxwIN>X-AY0iPogFLq6%5kZ|_$7X!qpLy&$Fxf~?O zmgGv~psU?Zf?=mHLF(7^Lo({eeCi^q?kZ+w?IkN5b_Nc7Qik3gp7!MGi_$X^K{Db^ z(wu7osj9U*lpcRU2-^iVKPR{T;>yd$>N<@ZTXUwbg7!dwSAvnRi=*!c0I1m;=Ko2rYfyAni*9(HJW%wo%@%}_N zsRYSA9q1%r=OhVBX{*Up+^x_~0>vYva*Cix_ryEaf$Jqa9GRa%(!uYEt9)iMQvS2z zKZIK->#hKO@iupT7K2>Fjp6~5Gh4l89}bm?e6JVEbTQzOxy$Ai~O*kcEwacN|kTWJqhP9xeR`(x2Kn@qI_Zb=jXZQ-bQrV+y94V$TkI*9=lV#H_@M zgTp;Xk&0c{pkZmt;pIv?toHJ{C(FDv73o-(>nL5h5$?&$x_q&Sh)8~)n^fi8ywGAT zxjvPUd-e)L4ZiH=+qOB$zWouA<|f`IW|Oqp`gLZ^=B3##dz7La_KaEitFq3O4|w0y zbD>u~<8n`h?QYrxE%S+*^c<}P1|BniE)Pq?K`r?^oHklj8;*KWCl(lY#&~m=C@+(z zr3i_N?m2mKU0!~E2PW^CoWsC_qt{_Q?!=MyikK?$@^+x_h{NedYF5;z)S(*db1Z-Z zz%SC8*n==bdc5mrtKVH8h&A?*lcjemsjgrAAk_I$>AF~5q2X4q1Kx`EFB@2#s;a9I z`^I%Y-2xpW#;2U>!miu6vKT zaq2CtO1il1%Wcc2u8HhvHLf(P(iK7^h0gyN8#9yeUra$wK@`>yj0a->AlN*@fJ(6g z5VhgnFi9q|ka3==u-9G~MF0Lvn|Dikg?0SS1&P?!@}s}T2kqYA$`vel_H0jBf!yU| zq5*OyCksNovUOLnBF$GwtnLX57W@j-avi&WAr&S6O;A$_z3S7u@hpds;n%=oB(9-`H{UD zJIE@Vy!_#krK{zgw@(bU1$uKkGaz}$tKN99C6^ue%D94|=JBXF35mE0dpxU?+iw(Z zxoJJ(tn{&)Cz8(XgzL=-tI__&ZR>cNHu8n6HB^mfS+U~%KrSztQWF%koI`bYz{>{H zKOac{OXt#t=Po+J@1re82dNfvCwIj$-R2cXcwt&4vbwP` zw37z70M#LlNeog%yR9UzG=aD*z5!o7f613JFO z9rcIJi4-!|esF**rIe07bAWy2`J4}P;!k&k_*rz;1?%b3?M&LXVS=%2pm_gR8CDr~ zzUv+1o=jcrtlRcsY16}Hir$oEVLJ=Mt+i+==O59WdmNT+*SsHzG9>Apq6irnRILIz1{l8i z4gwoW5f1;Z=TkoM&y6Od&P~4BcO*~G`^UX9HL<5f)2~l$ZhfMhD|M;Fv&CHl!{}Hl zR~xE!556ep_Ch;`*VXfPSxxoD*qdrEuM937iiSE>F?CW48BQk! zCgSS)g6`a*ryy8RHNOd&mmFlTgdhkg2B{!auLv!Lo_P@knyC|=Y89=k2If~Tojjg4 zlT|Ea)Dh{c^T?xNPr39K@$(xU^LYq&vHqe z0K&eMpVY@beaZ+rM;aU<8Riok%ZE}0vrUa&`ph8n#;L16EoDnvRq^pnzMMaj4m|4E z)_w3?o@{Ykf1OTlg?rWHjG4^MYo>;C0U;V*y9XEX_5p9JgMNFcemE>#m-;q+?_<-O z#T?xI-c~IGJAZ~8$Hu$9axJUWHm!r2=Po)rHfPa2fyA!W{6g#erTIB02nJ$mB!MK7 z^OdaskdPIW`X8H?aY!=4MYR1Q6uV&o*|3S%F6F=^wW)_u%P8#1hkQa+Lr!jyy!e{K zk6M>o{Z+ql&Lpu zeTQp=F2>c!+`V8D%m)zRibj3snJ9}8&lKS~ldg>eJEBbO!fKSbKPk9Ta%f#-1U3Yi zhj6GrqbAi(nrYqB`7ls)m_q6UDPS@2#9JbiJB;UME;8p7<7z1sY`EWABsXrXoU?i( zCwBfmUVbD4y)bJOCqn>Gf|R263_$JL6(IM$2)hSUT&?55tpFj3G|PYH&aGQlkb3r` zfK1Q9rv4@>;o-xN5dlwLiN&LaU)AP-40xc@Nw03-ya{Oc?EC{sAll&X7AcR5SOE#c z`S7)@4r8|*(;`lpOc6OS?iyz5Xt5#7{9zTAJ z)&goUO0=_o@clZ+HU4|>?|kPB@QR>c`1+OvwBK{;D(gjw0d@Bl_egB&h;IAQ1K=2m zeiH7N$Pl7z>#{kgW5@10vDj0{x3$I6QvlMgO230(dNdmMb{3-kz~IE}i=(5Xn`=%g zDP1m;*(@yViBVE2Ds+_S_;^}eD%j~&Js&=^=I#C|0X>Te0I}dHnKh~_1Niz8LRW;TnrM@TYjKR0@nLYI; z+BN=8qU5uLhMLUh#pIZsdn$G44%YE1yL%B4=2qGlw$(rA3%~y@*KT7B3Tq#nooDpB|@$G94#x_ZNr#egpReuf%3E5mT{if>8n{?@f^#1p@`P==+ zKiiqgCP?(Bd>v1TX^6)E8#%yfW6&&PuyjY_A_IBUhZ>UJS0x_IfCUS1+o3I zJ{JG6+`3?p#?8vQlp|Be{S2G&JNIQDo5Hg)FK}?We6{zM5cKJW-$97Xc>#qZdA_Nn zWS1)fjxl3*4T&80TeIy|O>N5#gu*x5H<@F9rXb$(teyGw(NSw(37Ng_?T^esloJ$i zbAhBu6)RDH61fRTRtBUdl3{@vZ9WnA=7ah$)7nbl^gbt(!&;ve3in#fXRUAEsuDo> zcKo!&`lQ(wiQR{iz8)#O*T52@jjDn)mXAt#>9_to&cnvG-b$dG;dQ3qiFH97_ z(6WwOaq&#SgEKl$>hEqls~yL|x?$(fv6UON?*9@7!y)?opl!wv3{*JI-jA;CQu0S0^$?C5Ga z`^^w0(nWvw{0z2uTvE{ETVE9U8?RyyxZB&|a6G4*hm%iIUqAMR$cL6#*o^5(eN?MIKb2(o9!k+_fiSoI7vpUo6H`_rV7^vbUO* zmGwh+Poh)*9J*2S6;i<>Z@4+@dz;uIJ_x_&eyz`V-KTdE4!280&V!Ea0kaoY>p; z*z4RURtE#$aIpDr;Yo_%kG<{Fu$1phXFo&m+@CVEEOJ^jcaCkelbhPF^MY@-!?c|1 zB6r%*$0|SXy*G;zrKf3?^VrsRRX5Y=>%_z-`3t<%@&;V1?@9kUVpmmO40!+k0P?~( zFgoXe_xsJ5*cR)c2HKU9c!DttLxr>yp4Fb)?*6p)+Wy<5~@{jZ)Y$HIDjA@L0=ddy&>@d!J{|LiM=eWFR{ zpDHbX)WF%4YYDR!JuMm>?iNE2y#*7IlvY}$B6s&}KX&_Y_MdCmD90l{9Fcj%{*cMX z($e;0`b!(Fq_6=FR-CQGJibcwIEK715~tg@AhT~EHZY4B`aQTOza}H0xE&MxibxfJ zn!4`WJPw|(fy&DIAH%B=;YDhZy;>LRV&6 z9m_W@B<@Zp9g+0(xNU`kfIRBeQXrJXuNLim{viZ`)D)a8_kUJD<7Jpr$-eqmDJqW- zIdYJdGm=hmE^sn3|7%I=Q0%H{8%3v3D)-qL#*dL|Xs7Q6aui%24AkfU$+`MysdR?= zMb6gfa|5|g&?IPRXnf0y`mdgbHyfS-!Wah4@quXBgN6P4)OB^c7$)`v6-1q2^ zFoWk5&zh%h5qRHkC3FXV5!x}~@FdV>!j0=hzM-Ab?hKpsXvUQd;YG=Mqv)68BX~0Q z_6IJvkSeF!^I?}vo>J{C2fh0F^*z{lSzNA|*kDh3MYC*&2Qd--nsJhO6rd;! zY$SLX!GVWu!vIDWo!h=EX;qZ>5Xwd>3JBPf0EaRbw<_)JzkcQFRdpl;{gbfjlla0h z|9LXt6aBp-hw31ke2(FI`3fF&uP*I>`?Rv#)1nHKAW;jA#;ua|Td$pc zd*|YeQ@rJrwJ7&-7wR=CoxW}~MTYb$onG|LMfK{D2VdA=hfxzRJ80Z$@Zi9sb_{Aj zH$x^g0wEo5Rp-?M0``i`dPJnRZ*qBAS;xJzkxIb%F~G|roeJ-R5Amf~FqKPI`U^1K_*#(r zc)c*-8Ui@tRk9sw<-+PBE-RS8mm^Ar5QlkLMp}U7Ahe@FMNEcb6Y(#Sg89fW52S;V zf!6?GJT)GakbzKuGc|`>sIZion6!_qDAr&T$Z!zXDzYk}YJ$?dQ= znY;`_^KMi(_7z^V;}|ODDds;jp(P*4Dm=btN9IpXT~6(k9Ic>)r3I0nc^qsREsq<$ zd5L0!e^w<$|1lmTnfORrPRv>)f%`ZjcwuSAmQ@N(@)iVJWo2J~tHKVWErcBymeT<^ zunc|gNYug{Jslk#5<*D-J`d`m3k#1U$6@Gu`_T7m_q7=GFKQC}T%;Vn?6CZyw)qLE zxj_r})RWusx7})JC3`dP?6;mU-=v?|ZqszuGBEP{yDYPHPZFtbevBv*yPmC^*~k9f z%lrk+@{R;Mqtm;Syh}M)5q@9lK}Ey)GmSZ+{`^R9TwrIdDv$ZZ)(__;zUTCdHrq8A zNc#DywIVo%LO~4^D!n3P<^O0UmFj~!Iv&_V3c`}#s_5X-49&+B#?i7Ve>c1!11M}f;U~1`lEyF z0hcsV_+LC?Tl*$$)pfs~FZ|_C9zPC|r49L=r>|0->Ns+W_aW&i!F?F;-v-}OYW zxbOW!+a_1&k{GKc3VHHE^i#=sodx(3R6MEhksuoWFQ0XOn zB($r1%WAs03k^I|4#z!#Nu||HS!?X{3oz{}sxb354(r}?o9PB%N>V;SdX0hOiZIrW zlq?mCMoRhtZzPM0cXDp`@x{nqnF{zN2dtYhwXyL_7J0&}&!62l)i%y$J?XdWa80tu zE2c}OnI$5wejj2a)GwN@pC9^h^OMcirf-rh%6ZS-W+=9dW0vG8hha^kw&}v428svZ zfg^JVLZYGqk?DhkhkJ!cGm|mRi;72)p)K>)Swb2%ab3-x<}kf`dndqdek& zxQ)nYlKr^X!rdtS=Kh(FQ?`2>9ha;R{n$|I?6#NjYk=RinKK_xg)SyFjJK2!Zp2~= znGOlCjsY5`AI6Y59Z5gWc1W1~kIcN-F*jUFlwZVOh!njJxGqcab$9RcN@ncIVs;Dy z%1@W3Y*-lH-20<5EG_V+;l3&6=}+1$!5{v;cWdESCe+^e2#GKPl!6HK5)cA#;xVE7 zI=6i-6l3V=JHcI%$PxrVKywuUI#zaAj@PCA_ZoX~xIq7aP02ad;h8}87nW~To-{e; zoPR!Fdnv0>wLiPbD7g8l+nbwHY!ijOe*Eq^`bTOH1nm7ABcdDsvevzJ@-jni2 ziHSw3JZ@LqfA$*^Zeo76ZQ=8#)3Ea9sK0u=f;H*{ZL-3T7;Ec0vYcT;o0CgE_$NI7 zdHi9$c`S|mb920>OGm8no>p}YFDe6#dl}U?fY*3vVJl*Mh%{A?uDuHon-3C-f-y`#e;+z#8@m|2w&mG zD8cNGEy&xa->orGWR(J&I{C+;yh z?a^x{rfLCH4vLDhSm3Q(cjLwlokNEN>{bhGgT?y%X7qZnYF)ZC{e^wU_=khBfg*+l zsPM}@6B}w=?X(p-Jct^EHg0_MV;nQnsDRe! zJUVoDhf_*#0opa4CtR%u*U9z$Kf1mGAhy3l2leHl2j@Z?I|hk zG)QGjLyLx%B$bxXqJ$Rhl%k=%hu?99cYNOO@BN2()&1P}bzj$c9_KMWrN!W_U!Ep7 zUn|YO`q`)elVp)*FN?VQC&x?rS_CEo%J-y&UdZF}ij6hQ%Z7Afx$+P>Ujas5J2{0mkD_s?AERd8 z4=x}owsv)$t(!nGi6M*$&Y_UJR4&@i1R8u@nm@!)=_xCezi%4l^ril5K@|}_c83bJ z?$ZTa)Y0)h%Zve!bpH3{`4aXu-D1~Wwaxw6-SgXI>91b(qby-z!sP!2G8()wLgE;2 zv}2$UXD(vqidUXc${#;|9{vV695P)Q@Eq*Um7eC4ltd;U;sq}Atd~4_f>GTN)jXol zK7uY%jKe%MqaOn~gj-{7RC-m*^veOi;i=;=mw1K>3kBZP^I5!CD{=mwLvJ zRos`!c_+Ji_3B_ryNy^@+W-nH`yM=x{k>9DF`gb~6QXNU$;@bd0Pvi47*S{;H|yq@ zPM(t}n9(2LLoB7J6!`a(D?U50)L5G5A$^Sc7s0QQKQ9(kH-ueqGQQY*PiC)h#S*fs zl(adhHR7na0KPL)EY|d5^g}7HxD02o)j=|2!Hx}}{G0^AgX~}P379Z0NsUq1DD@p# z2>-L9Or$4Ty+}+C@|-V&n%wHmthWo|yP7_dBTj&a{@h{JJhS)eZ~wkZrS>(v_Ox_x z;&{ujq3v^Dm%+9xLK1D!>g_T6WQ1pH?|d*%+wgl^rx1#waZA1oapy!qMtr!*RsfF~ zhTHp%np3i})?x-6Fgc9sln6f#Uphz%tv#C3vm;d`F31;>3X&>Jz%tA1uWLx&0mvjC z5J+HDcKzPH%Q&UmAROMt5`YJV5z=e?D~E)FMc4#Q7i}J)k3L1mz%wQ}XT*Qy8c42KRmLJAI~%-xecOT5c2>%OguHI!=g$~*QX-oj z>aGvLPKmTGG%=5P`RsQ5)hp8x1u@pnG#e6Vk zT)o<8uEeA2c}Hg_SlGw9x9vTn#k{9lD_4(LrJs9#zV8a0PuLN$CFkC~@luA{=G#jd zWGA@D6du-zmwDnsi~j7m)i>Bw-YNVXr^ng~wlSmra-S{fTPO94G?gKqGIpjtyGC!F z*pu;(ZO6nLwr~U$1;=4t8ID;IqUttd$+v#}>J5YC`f0Vd zh4ZILsRAO{fCO!^U)#a;XWLb=J!Z7+%Vb~IKwA`VVgt2Pmcekf&bFi@k?jj^N|Jjg z#=m#Fqhn2G)8wR$I`?xAd-+`0|$Nl){w@7SBU1RXFyM6DwE$PoIcwFkgr2Afc z)#U9ql|%|GpO{;`RqzW*fUgNcI&}ZtuU%&W)zJ zodhF{*fz5=39S9Ij1I&&edv+6rKjaFv_;1%DZ7rAC6(Sf$Z^4DJ_ji!V>hK3_o9pT z7c9mjG&2X!)fhDW94$gu!@l--54JA=0W^g8>WWCIMyH&$$+N4W3pkxwjEvw_zGseXKtZPz@awLgE^pn4lg+he3c8tR(_iDA^GbzYz@>FyHqDZa9q%S3Wxrukz)qS8x62@hZf8h{Oo=_%s|#UNq$y9a4-s_h-lMTRW9&$3Hvl z7SP-*X-#R&kzQnxQ>G?qVzI`zdU9^C(p-9G(yDYz`rPpAu#@a*9nFH)A4U2FS3KHi zpK{ss<}A}8kb#PDumz#*80l}}MB{PFEBbb8_%>ny@ptC!M-uIq9&3Utc?r4l}3H^6C8#EFs5C@6Mtsq#o}nxQ}%5(X{%jN<8SO2SFMmQ&yCPc>)T& zO9nMvf7y=wu{1OR8=0CxZf0BlRQDE=CQYI!h_f)FqgGL@Ck9T=^yvTGjhJfH2-yHq z_;_;ddU$v}95T^J-=2_)4E^gKR~)(R#_^C@Gji)m_DherB__w*b*lHEIZT{3ep+bV zFTHGY0BfV+efOIV%8QkBE3Qw@Z5U)Bpz`OU+Zedm?v={D=;n|mkr6+xnMq4dh4U7K zS#oe>K`w^C0B;hGm*m2$&r%GgpUJi7{Uz0V;B}7z@%N|tPNubllz_fOO>0=QFu2c(cQWT$rW8tmmH{1Vfk!~K<` zm=<36{_yT&(6vxC?`7ZF^cQw(85||Tu2@hIn3USEUe5Axc;}^kWt?x>*^XCzmTFn; zwD63Z3(XfQ`376#2x5vJT;`%^ z33ue@0{VFg{XP)^1KS)dy=xcB~kulwloB5ga#qoGrjU zB#0M0B1|jeV%HE;9*l8pdDa^IIN(G9>7%^Rv7Sa=t6dTGRU=g;z%?KYy~YNzqe8=CF8EiL_c ziu&Wyx9E10-6Ui~s+U>5W662F>(2b!S|7>vChLDl6!TWVO3fA zs;cTpv~lT&KR4pJ_R%@cI_5mRWl@jfKTId+I=6Aiv`Il~K*V6vKu|+NU5=l`=QU(a0{_NT$tNf%C??|0>G<5-=NEs>vln<-H|Og_ zi&n)?F666${&oI|ku-Rq0RNk4aWxX1l-Y(tz^Xrouiv^7!ESkDq^`p=I-W3z8KJ;m$qwNJR z?#)-0HK}D_9Z`K-=z)t^J3(2Pct?*1USX`z?3gNi$klvg3 zcWcfsiN=2%1dSyMWw)I&t1j$$Dbu+GW%#Zbi}uKGy|30x%7$yY6Zn5qW_h<5!zg`G zx9{|eS4V5zZkX5Hl76=DKJR4WPl(c&1t=NLjd$EUbj8vEBW^<5hE{k2X%Orv`$J*5 z17g~L&l@DnzeEXzzu^6Sq~Mx1XVCl*PEVHo@$tOKjFP$Xd#}p_y`!mXTEZxnvMr&9 z;!a}po+QSnvAMP4Voqz~rp(8Z>xDN2S=``5S%Oh4*>Y$-De{c?k3i^tmOXuabOB0U zmo8B^+EvOBhYU70fzL<4p2hSUlm7oX2ERcB2r3znA!Kn91waGM9{S(?@Y7n|o@H}; zj5-TxGOLv2=;Ke6MmKoq%pw#c9)8V3)pi{Z?N<~E`IFCOZ+zf`Ex2S2G zFg6u~!6$X5@ycCgrOeXrsoJ5nJD(+d#Zn>`Ki*ulM12iM84?vum_yMGFwX1)v+D(z zE>sja$!@Q_sU)CpYr7o^fd5|jg^>@+#9r>!ojYQVE$9Om0E5j`(CFa*RbMHay_sM6 zj;twt9NxXjC)BJvtQe`Inq5Td#jMT78N+6#Uz{!FGagpkoNMK>SXDNhohidml&@ux z(a|ion$!!!aR<=(d@Np7AZsU8e12K}XT~xm?W#|)wS{ft8X!O5m?Dv>z`UHj8m1vF z1rG^sIm8+6F)X+V^N8yDjl4yRhl?(E|GQ>AQy%OA;N%n#G~$F279ReP{xX~2ug%Js zE?exDQPjclP5b9&Xp3*KT3x8=icm1>q{{MXJl;Q-uOLe!qOF+QQ;;y3f9?1sQoR=S zu0-|PzvgNsXWueT0XM;GH*2>&ORy)1>fqoYp|Hk;tP-sVlU()T!-r3K7c;}Td6b98 z#^8^f`JW?>g(ZLniSxU_o5P?l5O4_v--Lft+MMz-Q<-0{bl{y3GkHd}d-iJ`7jr1o zc3Lxqhnal{+qNn1ade?!LRjPK4`Gy$JN@BZJA?-c7WY`Fmgj6?0ZZ7xZnuwiH*Vuw zz;#Fg63d3k2tXMa{$Tnk)yuegAMETF!U-TtVr3J$@W0Ci6Jrta&ek>5R1iZXoh!Wd z>UUQwAW#QH|CWBU-;HHkA}eR}3j5yPEbze5%QCJuy|`|H$i@~=uGWf(a~DQ^g`STtxc=)`A@zGM2yofht3y)^UeU8jZiE?H> z_Dy*9ZOh>3y>p2r#{K<)eLH9(&+@l~3!N1(GQVj{;AWWf!n^rQA6!%3_jb~h9fFtM zfA+bZko64=51S3Sf(FLFdGiy6J%5(IZsxoI!GQ$_^Y*{ON69VvULGR43{x?SAy>Q3@@{7U7RcPY)hC3y?*zd5cxn{T?MYrroFhWgwyRcrYDc>x`>QRB&(7jG?36}96c&( zHoNo&pI0H}h3&gj+{v`&@#Y@NZ2PZJ3RH9y*{gYMCd zzjO0eVl^-Mrp+nO__)V;BPo1u(+noCo(Qj==!nt_{#ejf(0ARr)WUofm?4806%wba z*PqkRen1o;h##xLO;H3q#K^|BJXzBp1X(QiFOV}56boF6Ag~Vcq)8rpApN~wUYH?J zx^j6A{8b*$H{vjx#P@I#F>Ap~j8dZt$u2y<7ryP=jGOva#!uG=Jhywf;T*5@bLmBO zeuHWW*^Jh9sN}EqaCjP4HU`>6%w9=c^mR>;?{wD5o)M~VoNZfF+^fxPSsV7y7wk97 z&?d3~a=920$<9p}lCbR2A3U@Rur;N*&N^jfXG`)Qej=X2xBKtD5?$u{0H`;`F1Emp z;3P~o#IE|V1^n8@uIZ#4HKjwR?Gt@7)lcq6FP%l4)`i#7RD3Oh^noc~_g4tLdc#vT zu~C*Cgv^1eorb3>)yg^8ysh;GrY=pkmdg2nc((X@hzUeoV-d^hp+ha-06 zj_>=Z#35?RM7)Y&nTq06EGrNBIK*fPb2=%mowiKSoc#e8|9bX+4b20~GYmvtXgRHB zsVqf7aNd)pp%e!Ajt4`=I*B*(4aBC?h@wx`mZCx`2on=Xxk@%jKDi=B{3`^k)+@t+g$Vpl42y?X`Nujh<6(?lF%RWwd z@D~{`w3u9YU4F(c`tl#WgRyG$>1|83vbwm}{Fq!tdCynAIESNPHK$5l z0S{IUVsc=J6mIf?Y+xV;Kf=hzj~TIzf!DSN{Kjl!y)Eazf80O{Okm3YK<%8+$I9H< zdfXFu$ML2!7==vK70h`9Z7A`$epCu^f~O1(@jdTt?1$~L2iX+>eJYSRBIpXCT{lG3 z3AtayrVRASXgGF^a-0Pnlr(Wsnu&xBL{j!((eoPn$yKcBd8d1fxDOS{RDC zcgyVpfz99H+dWx1z)ytlyKcuB13-mpiG4p+%O5@3OfnpiEh3Np5V;sN4>l^I?Av%S z*qvAo6ITa}+R(BhCl(BA@|lUWldw0z3QBs{pX)Mxeg<}czS_yf1XV&$9>Pha2}CwT zKoMA#T$r2evicEvp79sJ_Nq2}&4>|y&dj3yf?Iv(>i2!H*e|rz>al!V$2y~`tXsxk zub)nIP^~_L<>AP?E;Xp_IY;uM)OKF+Tt0u^RXFJ|SNab|bdHSN2t0d`HBvKi1xi~W zd+!v6xc9X%szU^`aM!LbA`j|^jjrExgikixxIXQiQJh1dHN5J{PG_hl%qL35}@Ms6ntVDyP zhAB{(dhO@W>xe#vXrxHmSr%X<+zQ#LZ#+3Au)woLB`ib#8o7MQjZmx+a$%}$Bmgb8&|@Wi9I4-->8cLtSr&ad7C+2SEKrV=5=G4b$g z<5d?lbZ4V%q#lK}*8Vs-5j43D80a&rYfG&+q%v9c=J^F@j3pgVllKw~c`G7Vm($GPr%>Es=TejNL#ZX-#RlRi-i;f%VJ4-nK6KwIeEx7= zdF0*+yAM5Al=|b&TP+)Zm)_PqnfgA>Bjj_{4tWyKAYWsy7(^|ehrcKB8Y_uH;H8Rz<>M;D3a&Y zZk@IQl=a`~M?2mGlJN{9rO>AlFmfV10$5Ggs9yhE3|O|DQ+dxGttLTfY29<2kyA}q zz$l>Oh#6}K5$mRlmnZo!{k43*SWT?Wk7@V{?hmjZb{W^5D_qB)^Yr6}Z;eJ|eS zYg^+er_mjo7aC!?jb+h{?IcSr$DyLm9VYxK+q9a*+O#)~>Axu)Q1u9wkyv_Q?1X9c z?$4OZo?*B%-oq$LM|Lcz=Ac|6kU!oC;tdO(RRAJ&iG{JXgG1&gF>rZs(7Ym;GO-PZ z65+y786V+uO6)Yp>(0&ngv^*Ak*}y12mKO68L`09XXxY0IqRTD#M~azRCO&aKa5t- zu;X)P1XXF_GXn|T8}p1^`hs=(eJ?-l#lL7!Vi(;E6I-6DpcwZD0%c{&(|3Euz(%5u;&z_35 z0a1R$y)D7)j_>}31)#US>$O00L=f3B=k2peN73^ zL|~56!nAMSKD3*@77L%GMMtw%oe~lGnq+b%F=Sd~eC??clOzby0AfE-|B26i?f!ks z=5e@9bc%{oxFCp!=Bu+3-iW3jBqWgot9oDBWKMd=j;d2^XU?A0DYp;*rBBO`k^M`b zCgnCfJPcV{WZf~LH`-cSgu(jeP4{G#zrR2JSWQolp29C6;0+y^iODGEcSuHwBu!U$ zdF1os(0HohpHQ$}6QLNUt(nv)g~sQ40g=Je^-d9pa*X&Ct(OZ)BHx7UzAp{p^funQqqihq4T$HqTIt-qZm!g7G)DVtzS<@znH>*DiH zQ5xglK6=!>C{(4ZTH|hPnqH3J@84POBuY8a|4X!%c>hX5ylxP~*iR3gpLxveEsefp z9VSSw=xq0)&llw1njjmo>_=dsXO5IY%$kS=>M@!p3!B0Rp<5WPGmSU`#oN10?mN4l zox+9OwY-QxpZR+AL!z{gUGsA9J?Ax%oT0x0 zNP-7u7|?24-W-D^1}72Rpa}iEX}hfm;vW9Hfpc7Fr6LZ-aDq|P($b0<&jA|*GF~(L z>%AKe+-b$0kumX5YiX>0FckP{$QNs(8p_f9N`od71Z34YD?$@s81StczBTW zn7DJ-IUY>bB-sO6&(9@%%ZCa{0;8$KGX?&B-7LTKXM{fo?$KU$dJ30I@+vM}w+}~6 z7Iyk(2jX5YO+S#-SV8Qfz#MK@gW z_wTW|1j-Ix;Z@qnwO<&rU?MBZ;q+}ig``qGhM+dFWCB$4f!C`JQ79zJCwJuIfq=RE zgBjg_RCJuGgL@vBL;oi3c!jSkQl6IWq9IH6%)%pTcd|H4E%+OyDm^cjZ#8agsNbnk z^xlZsul^PJA+l#bh?xe5Xb4bz6&Kdq&!a|E29>b858vdmo-VxTYZipDhl^KjMk ztgK+GwE5$L+p)B~{570SeUv#IsR1(+)KeHjH~2jgGi6}iN--at5Bx+bO&X}Bgw+YY zV1CgEv*^88g=--&I-G3BE(1w->?TgrA3W!;PE|YSj@huht*N;ku$bF+Z4ighOVLx( zjnN#f(RCFXYMS0EaaE5yFD#|!`zq@AbFZ90>?lJ?_1ubYOr=u$FUNnizCd!FTe^j~ z=q?8ZDH|9>2Jmnv1b1d8_BWY6kp35Vp#*km`H^+d_8pR)q$VTZylJS}34qk}yv!}C z8X`9~8SU90UogT>w90xEAHS{Amu>axqg`t^(p5O>+Ub3-hl|L3y{02;s3`czw!MwT zDXt>$dvD9KcXN5>=cONl-QJrQ9FW0xI89WR{p(b$N>2Na=)wnjQGV=<9#T@&yDW@k zE*IloUeZohf*3hHNM2y!?=zhwsewwvvwg>o6>Myc-)3EZ&N{G3+ET-@^O@nfj=h-f z{TJb?4wHPAmR>4$M(Rkr!hWQHi@Sy0xZw#o{T0h&c2sZb1MXszDJ15Zt5>h_cM%(v zuzgM7CWp6%Nn3Xrbqd8FZ1KHvb@qMNeIL8`yRvG7qc5}d1eJ3q$>_;=AJINJ>riuf zU9B*sYD&4aP%mo07N^e`ro?p{)M!bM0pZkIaZ5IE6A0Qxx`Kf2a=qLOE0B462~JIN zuDJ`Nd6kXJ8(;s=Ljg`kjj#)YD#MMuysenGiOHO?Lu_^9%plT+lJiZv!NG{0rntj* zltg-vrv>BFX9mvz#fQ8f8*f;>ssrs+eX`*^Tx5?wX zd+nsT+TN|Qd5mNtIa`)ky7iIq$NbH!w>>$+b!zdRR*(DZtsQL2BYsFcgOAI+3T6R? zh_zAk5>5e39y(L&bhT19LYF`^Mh=ZOz6$NU(;tmR{`JyC?xa>|D&8a0aHB7Gtllh( z==EG1e*>DGB^H6C>?GtYZ0f|a7&gfZG28AeiMaSf64|44aIqy!n{UETwFP7+S{D+x zC_=X*WhzXHf;r_Jez1`ZoALsX0M^Kv7OOW(7i8ef&@f0H!*R6ERl#V=_1kb|;do$W zPgC=T3DsQ>JD;l3-#Mb|5fGPh3cGaJK>_2Txo|J(ZOLxbv5hMOY2}jHxsy0coGnQcRuIvTTLP}(3_)Y-N597=L9VePAp)SFKWqhnMGVvlvjmU z4B=zbcW4_+WTljaout$&>btBha=))_VTn$zqIn6NjLgQDehPYXHDbL+=kcce?S^t;xv_N@58l1j7XN&Vwfe_N zYdUr+d}Y04onSq zqm)45@_fmgi49ECi|79qVpXEF`#l8|hET6OZ$;cVbRBZXw^6{C2tsU!XdfTh$ zKxH`}_V#uCs)UKT-D{SsguN_lXvj6%YIR>wzng_F@ zrmKJSeWa@w8$o zzngQ3Iy`(xa?BcH-yXM|D<~WWl+W1UoLb!-@QJGJ!c%<-&ph2e#`P=IdroH_%K!nX z9q^zh7cHT~6L*${6mo*zY?qRh1k-_*Sl1H4ES}Nlj;sy|l>dt;|5Ek-hO_H?QW_SJty(zT!h$<7cAmgPuDFVOAafq>jjlsLb#P|>lwA(DQ_Pw z|KuvRklcn6|Ifx>maEc^SQi@3K8myMUe#$B!NtC3$z-f)xhN=2EKGzW2I3naGY*## z_2C6u(Uyya+6k>+Plcnk2wf=|>p+Y2VPnY#=mVGkYr7^>ykA`Ihn;Zl3VKt%u+uwv z)DhO3s`*W<=4x4&mS@gI)~^$k_iA0aJ$Fj~Jy(2KN1D5>Sw#jHt;CI6zT~FX^qcVW z*NL0Boefyp{(KP+Ar^kGr+LSc5zcL&R2yRT2$8ngX*ymnkpt|h*5oucHumg7`G~#F zCP1|Mb3@EwJMEXE?iG^}`+Vs1=?DKi2K6i*#QUJQX;Mscb=3A@uFa|9ypL=TQF*4l zJ3lpJx9ye@Ou3#pZ4J1{FCu%wfIo{8F9IVd>sk8CZPq4stQx96#%44_k~6+e?bq8R z`%_>w4#M4ck9_QywOTS>_;hZ?L_xmAif^B%Vy#UrCO7s3n}L}n%1?ax=DmeUYP5d| zMi0ckq+{L^++PxE<$@Q0jNq@ka%oFY<5^4|lHu9vF;Fr}+&J)ol`rgO;CO+672WWEJ84uz40W%GZj zzEE^-m5q>g+}_yOA*{IMGbi52r+1u}`7G5q!>NcCVj*iOS=^(P@2o(R-AeE3v)QD&Ji07<@IV#$9%Tbjgb1Y@gLEiL-LIZY2A zcIkZWrE{6f4mDMU`ZJm%+aR%)GfDUkePES>!#=@yK%Y?joK zChpnmxR|F;OpZuoOC}8Ns#JSTPg_#@jmx;&Iep8D>$gr1HW0NhnBlOcSJ%*>Mv4g% z*2%hgZ!DrYE(PZaGBiWE7X|+$v_$6(s?VjPSE8bT)c;>r1T_uKDX1dMRr#5)h)Oqr zcCkx;VDaXv@hF_Go433AsM*kQ$jmM2pR=mTNi{1SKx=AhQ93Hc>+@>EX=)KYb|y(f zFI`&;+f8xQq$+K)O31o#tQIe%S(h(Lj0^D&Iupj>eT*mHZp?JPIXhpN&C#sjSN(=g=k?AgQ3U5-H^;>n@Z%gW<#jdXLk8$Jr98 z93p&UzEutfkicrxDy=ZodTjHHRm6Q1;&Jz=Z!7U>Vz}{Xs>WYw-9LUd2d6F_$J-63 zvx>Sl*SYsN>|O4ApwwDo@|^lw|rq zf#0|eN$vUMRJy*UV=dXbPty^@PeT;&2to$XOxfy6!7uPBK$Ndc{>;z;;yDXlSB6A! zuH*2wv+vl3^D#sXoTm8}Yya)NJoG$1{}N;b5HCbjL>wbdE;sw5iHR)%O4cq{361vZ zAWjl&_pDrOh61E`KjH(0)VIol4)H?kI3>eBsKx?MB(hJ09j$gE>q=Ld$ioR1>zop~ zYCefUQ~uy>aY%c@Czgmn+q^V@?K#HV5{~i5jt0GW97v^Rw=_+(6jz5|A^Y<;-R-;q%kC*^UF2RJ z@$(L74=VbvXWj);`So#n!0zPb+khkVD_3MN(HETJM64(VrXFCU4Gg@%48;cbiHxAY z*7MX%R6Krs9W*W^fuy%SV;OYOlKT%F7_8N?Iyo~jxK_eCGUnArDd1@fD6nx;`d2|m zxDj+%gTM~yiQMc5#DeH;7lSwB1yrUw>iR>F+|DsfN?Q=cT`xe+Qpeu-~8?2#0Xry;D zQWFFS({07Qd-sN006rv%%p}s4jM!jHd}Yb1jjfoCzry+dIntOk22}r5@Kk9~0T3n) zB+ig|JLG|pkt7Gf@!Z_l%1x?6FKv{?72e;C9QpDlRLZY@5sH8cv%Oi=gH|-p3{umQ z#n+}ySDGiuT=#@5M%~1?o=Nq73d_MCO=|#V7#Lr4Q*Fb*><$b7Rcwj48pi=KA;AVO z>Xb2oM(7ke_iv5aI{YFGsg0hA~KQ17}WJl*V6*TK9TcQ5;L8ZU)(XkI_`Xk(|H@;a}%Vkhbl||ih#$M{S z_QJcZQfk+Sl1IG~s;$TR9Z;mKuwKHkG$Z|Nx*<Z)QSeG*-PMU7L0iegMnkA>WUQTDUsADzVLajfOA~7;xn`p-uD@;)jUo=V>v!(d zCb{g`zP)_7;qcqDAy&c6dmdf>r(Q*X;pag0kWHhy3$|FQysny@fCJxXv7dUs9o-bF ztMKDeeZVNMsFhX!MvI@*H4BomWxbE6?lZz8N_77qEFWAew_tpn)}!`#)q~mZorBsb zY-DQ%shU*9#6t_lAQKZ4B2ERc+#~k#Lb(H;>@hsbU&0*l;{H=Q1HWR2^!^(hFiekC z4LAKIoQ{`8-Ld}LTdb$_=K<;Zhp1(&n?2O8C!hY^VaeRT0#hoS`etabrj=WTGA}{K zI?A$sy8ZWlh*KZ-HTi7dGz1m`I`ed z&k2KUE+nrXg(k^6W>r!MEVfY|-L4tAeVSia!FM0~64Aq^uIq@SrTF0Y(k+Re5w+`9 zUGzCM)(Wd9O(7*cZIc&AT3)vgw;YE3rNvS{xW~?=TCGJh80mJ2A$9%sLTUcructR) zKr+php86I;=)(7%x;GK?6~Dm1XIUFGW~*~7J?6A8b9188T7Rs~$}t`&@9RsO%dA#nKNh3ec3;o} zAdW<_1A|ZEsyOQ#ta@+f5v;TWUb|yV8kYs2mqW74z?&_pB3a?!;zY>0;B3RV#bU+A zec0~JC>P{;|4qdU0>=hq4i=befj^}T_6}Te&!4{|vGd7>K&g^dBVV^M@f{WCxUwn{ zOqC%2l~fAZ-&{BI9UAVh8|xH5CDrn<;pNs(vwUlQWE+p#c&Wd5DZ?Cfm?||OP(ETC zXJZQwwa&W-I!$qBr@4T+>eH0)f zLKihJQ^MZ>s!>L%M(`gH2_6Ayxj=b3)%=WGQ%OTca&pEE0G+8{7smn=I zdwv(1+K)7Xs-tycWek5lb+SB3blZC3d39__*p9RG%`W37LOH7LBR7^XvJlqBBQ4Da zTM05Eu?F5pJ|*!y3K2DX3?df7yDoFYpynpf9_UGMVp%Nxb7q*(=79zhUn-)VLJmVA zyF+6eyqG$_c6BYJAn$iWfLDk2zc#UZzc?DRH2qHZ0aF5*PBX39Oa%IV0Rctu^7O!?(ii=Hj5j&#~7Q&Wgj5B-B=UHjl4Rsq22QM$Wyem{*Oir*Yl#6z6 zg-}V_7DtH?-C3beZ&&M_T1=tjU(S?Oxfv{Q_9L-jqqsZ#=npJZccmTbU{V!W_orNO zzTO-9;fCe1^8eU)Bfd{qZVrpp0PdIIxyX^>9L#MPqD$`K%1Jl}$SrLzJQj8uN)NtX zMG^uC21x1}*!6t-^kGacxBdw9ZxlVovXy^?zeNB00JC#Kmr_8L1Whur1t#NMFi4TV z*mSA^U_2qYVNk_e9LVlZ+;q$MBFwK_Wks3p=Wu|pZ<f8iHHLq*lu>0=$c zzpjDpTEJb4*fzDC43FYO9#Sr!XGkv--q*Z;^T>oi21WViaEkbbq3fE)*i{T^_>S7L8o+x6w=_3 zhR)`%mjZa2_rGY$ZWaV*T`DwqvFebt2M7-0rV+4wH-x*$d|pVhP6>_Tve;WTXZcl> z9Y3CVJIZxMNaw`0=fWdJm#q`x`NpWUPW7{fhD}UaYn;83(@Djch^k`^(art)10rvk zNeb8T%AI#qJf7#2Aw%yEk&59^N7;GFX9DY%(>Q}{MS0Y+Ud;H~L|;0lNAhUYt%o}H z5%(>77ABPTRFv1RUmE}uwjZlA79p#hc(%b;sGSf7Ym-9*3RtK2>gyc(zJ$gdI4CHv7WAi?Rzu@$RN)u;!I;9O4BF(xs@Ql*AoJmxR4{TW7_k(va##$dYirDXpyZ1@r|;&HNUD>y_8idK=C> zjI+Emh?S*xZmve|^iofE?5H>jA>r&?L^D0|-O!$l&lQq2Ux97=b8c23PG0`BBC2^x zzS@lqf3RwA5qNqqG8(vqgiJx?303#^{+SJrDzuIpr_p9iwtv~P_R39G=k}XX{8XlO zI!UrA__S|oICm>rrW8V@p`%|Ak#vL zQIp%6&P6*qr?!!H0qo!XAVFi_ai*O3gHZ>>ew~re0jmdy7jSTN&hZWn4<7Iimx%V~ z#!rsofT+H>{D>E~loTtf^Y^BqbE-aYDg#I0fqDbJTUJ>~YJi#YHPMQDES}nOf3Fy_ zHqd(1wsq93*sRhd7ug)Sd+?sP z`BbCN%PBX9g_MA^i#5K^>E&@{oLy|3o5;%086?x7(AR5HmDS-BaJh2@^;_?xpc7>! z-=DrzQc0+0+PwO`%%^2u(KF?Ams1M$-p8}nt-WY*ZuyyJ7H~E;8os@EuH)|awnY1v zurwAnQ)7DR;@GvvtSBPt78&!VUG(CXljFoNcQJ9>zW;87(V!GAs=U*{{D2e?cimS;3t3xaSDr5NEuZ*PZ$ER|<0cW~9sb6V|`*W4JT zf_zm)MQ=wIcRX0I+aeWwyCqM+kx3nPk6>I-QD6sw)COPt9{RlXUU(Rz|`Jzy<+bm`PVx7gWHjASD8ve^F6E2c(TS_<(s$az;VJdk`B-vd>9W4;pRJ0Hnt>3h12{BwkSPl$qu^nLmN-HNPw>Cft7Q-*Vl9`bqM)ybo2cooqLjDi} zo|$FSI9;P^OkqQ^AkwKB)W)+U@rQEl#tlSP7^Q@QuuYPFK`5nMyK~0}eaEn5MQQ0S z08;gtsrj=~Ted928!SOhQ{-haEH9f0z(XRh$e zl!SPAmAg9yN(vND&@jEMtgM^~d;^IWINuG8j}II?h*QRQG8jXq(OM%i;nb=IlhE|h zTTH~L;`6_zojHN@xv0mF-=85o!`SB{;Z^gF^s&#AT);t0K;SgQAv8T7>+0&9RbiwL zR}v#xLP%5~e7*t!1`<@WQhxgPIo z4;CcCBybl7e=Z~mooD@oM+P*#V8ViQIMEh56 z+OlPbTcg_9^v-tNJn>b}6pT)vrnH7RoZiI46H}OzwAT@*94sc*PnGODoB}6p;%Cs* zbfQ~ruM#j`wZo@Eixd(D2_}O{-WRl`iioM%hN$R9hZYw?Xm$+6K6d)-*><#&M2SU0 z$8ii`%1Z)8S1KI}yy0N5baRTiSEP`owT_GWDFC^xQ|AuUvD}x{y0ss^e;99w)_ECE zsM=&Kb8`@$U*EqOOH%|eIw@D7qJg8CI1xfVlG^?%G%H+(B}tZRBh4sqVUd!V9e)7 z-WDI1eh2w(Z8x_|a#g@0p`$FSoN?QdmKojXwtqdc;|O5V^Sz%=RvBil0f}=VK*;=^ z4xw5X+X2XvY(NsgNXYY>%QY+SXN07iS>KGiU`x+(=|!4yNNz!$fYqz?t^=c%H<`uP zecczDESq9$o2s|)W%|OfT&ZTQu=*Sx5e{`V6Nw^4yMZoyuH?wvVht5XiR;?#ANM!3 znAE+~_m{6@U4e5Fnl;uG)xeyKB~Q<6jPp{bnZM;$t(6l)2ANX)AfgcwUVM#dl5X}& z#7meD2}86n2;=5RjMFFJxiUYtL9-@sM&=5n3(;`SCCw=jC}MhckJCReuo%5P2|lvp ziSA4u5M`C56L@-Jc(C)l9Na)HYA+aX)uY|_ms!<(&x#^tEiu7`GdKU%ts-ts)tH^+ z^Z#O7c~mS}X=MER+<_W@Ro~!#_xN zN4<1-X0~m6!|cU9i(eQDxtAT6TFJWVP4v^W214Q?Yg93`W(3-G#6D(~B=ZS4( z2uR4cFne?lh<;i?Rt-2>P%SB)I>n7pM4Xr9RXR1gUeTQb1F_jpJh~H>cHg{rZ%O3y z!Vh7id=E~AxJErp64)L1F9R48mEdhu0C$Su(dZ3o56X@LRYROF3`b%Q7Q({zwBGck zcrm<8UJVyUFRp6N+-kE+MB|8A{yS9i2kKBRo_Cq`t1|I^-*9q$k7c1=U4P&}(F=aL zb7n7h*9Lw>-k-fNNlUuf_$gU_5bhX&dk8;HOtRT?(Xz)*qox3W;}k{t48`b>xI;?4+E}AJo}!;_L#dK zU+s#WHl}Q{^LvQx0gHlb_L07CM@>Q=^#`tH{C=?tPinW>8&T1##Ih5mttH(&WfG=7 z;F$-j1Tw>m7x6*FH%CnJzU~anNht1UQMn}~KFy4teC2ab4ur6$HM3Zk*ivaJWbW?b z`E->|4NZpNUoF2jA&8yyizc!87C_BOWnoT~XU%CX zzAn}~2~$DlCoFo8^Af^DH)Gq%&iBg@qEVcT$}70&Zyujft$Sp6KCWv10bIX*+Z(z< za5V|p6FEOjp{QNIq9{vHZ#vUc5E0!^JJrzBN%7cYpeM$54qa`yfY(TF-;!Kh~ zR8fMgl+#|N>?LIP{-0r|Jg>g2m+ZBj)K5I_*oLbJCFzw;Y1zsK>Xp43yycv;^)&m} zI8mWfGTS~fyS#F4?QGm$TVcNUK$+-g8e*jVi#mhdPyt4tDnaX@JVaX!*iKJSKmRIm zwIiqzF%rIFnSQ=cn~>R{parmK8eRgg93$jVQGrPM0up3l@KF6r=x7!~z_~Eesb(^y zYIg=T8VJh!=-LXdX~RDZD1ulz!&~_^+AAnvP}Y&^3qhhzW4%iPV(n)J1NUC;xG?`h zYnW2uv+PLawAqka-dWk+7G3C0tMBi#$IqeXgt;aN7Sr>Id%s9ERaL7{=z`sgd_wG2 z`ZRRZ7x7145*K_Nj%W(yW9cnmU7d`GA(L(2pZ;T5^rA3b9?Na^i?`f+j^TRC?%FY4`;@Y@`ScigX4OWbjNT31 zhe{AKk+wd6|1@khGjojF&$(b>LmFaLfvx}sQYW!oP{9?$!brXUODzdAB>@d!%|`1N zxL$xPv<;tvz-Fj?X=!O+;J9z%7TQc)d=1Uu>#*yO0Arh>dhyG}6wUjzBu zn{~NOX_WQPE_?H6py&2XkD0Q@&zv*9-bo7^!_6AkspnnGNY$KMcD!Mg{t?h`|9X${ z7PO-wRRkXt&6VZUg$)}vkjM|<#MZ-wlDpvcNooyTJ;K>m5xnEy<>WpeXRkLwO~wXqaUCu z2Dj6z6*{)t9to@7JhEiCII_nTA!iDL$Ww*$0J<13jB)UZa|U09u#-Z83T4ADW9`2^ z2tG=j?_EnSUY@bFt<3|2u`ftdRfJda3MMA%->wTrMs1DRcEmiHBoU%>_>!F2OsY{~ z_LEpV7@og=uzFZzAuX2?O_cV~)GksX6-{YN(k?9;x}Wbj&N({2 z`*HtwKknP(`#r~LT-Wt^kJosD39n?;lxPnN_ji z*qn3Es6ol!`!_E6*L6W^iLfeb;>S5$V@@)(l#t!vZb>`>%Nd_`0$Q*La<8aFs%oP~ zhinKpy#RP`7YncGQAeSjuOU|TcR|$%39SxdT6c5#duSXWW4DMJD8Y88vn`8m!1zg> zJ}q{Je71!yP?Gg4Dc63Lb!ZZst7V@-k;zcKR3o+HvfHEiz6DJdab8K=%-si$jKOxj zs8aqpFBIn3#2L}S3^LP~FhTV6^|it2egD*g6I?EgC43uxWCH?)T%h`J5eW5OAPa%8 zgPxYG!s3|_37HhaRE;!iWC`aEkBXW}Axd+2_ zRaF|#e-|Awud}wYo)bmKhVp)~aFs80l{Jsnyxa8F8R5p_g5=)^Cka#l&@gw{4{Yqt z9M;lEv783Qp8Y{fpi;IUQjh*!a_;kP5qkvZdCk=|-Y)FZNm8zude*_4y^xrg2zeW{ zaN*F@OpZE9uiR(Bx7&iL@pIHJoSGFq6+*_@`=r}@zvakW{%+X!$;(#GTygKWG|j+> z@R1YJym~wkHU7Q2=bWcg0P|kOEbe9uLCPnRGXX)$i)Y@jlP))vr_pFczB>oaATN)* z91llOF7tCt!%tn<$xxf$w(}_AYi``Q(Xg~`F^K)|6ATz(0Jn~OgIOX`JlV74`FA=m zG{@*~+8Fu#&X@pK>zyXoHcZ1och|>*PAs*4_ z@lZOjY1hzjh1wRtEaJc(JXlR+ACaGmfO+BxDTM=94lHeq9{YpCF!>4?p3*e&qi3QQ z**1<@<}*h$!uEoj`Uqy~p5(dYFT(TY%_HvKjGea{y0Z@GdkJ2C*VlK^s}&2cwNET4 z9312T+nUYfmBX@z#Prq4e$)aC%%o7#68i*2HtJlGFqW+JB#`nKwVuIj^ z*cu@Qk=B$Al&X8DASV!1r8aT0^V6qZ0WOw*1ww)j4i3GLD$&Jd{)22ll&a3A6Wq~S ztP{%a#B;zk5=-zo5}cnitIFyag?txP$X#ns=Zsl#JLsl}#L<=(`*og^6aPfAJNNhN zj$D{4A(<$ND|#hRyj9fVI#}bse6hXAb9g5U=n$U*Z1Ukj`Z*phmj!p6^^*2=Y$e!u&0o_i}KZPq~x>)q&xvBlW+rf2nL zW~?j0cK>~=h$wIqlhz-b#TU&T3Lb(|_}0q12_c(mZf0233dw7qRjCX2mX*8KVJ9tU zn>pv)$hw4=V|_0I9_|RNPT3bfjr52O&Ara`JTCg@^aw7Icwt7Q}{&Do15#aUr2Ld*QDRL3vqFU9eG>m&`9X;IPaaHL?`s(?sEQj|%P-y3I z3z5Cwjh3`U?h6iF0XD$g;R-@3*;;KEe%O84hNSrka}J2E)&Y~~_vO5);`|5!1)xC( z@7ucp3sEkp7v;VC0?Ed|+W=$?h|URGNaZ|55v!1UC@i_YI$&wOW>Grlix4e(VI8B6 zgoa>6(Hv=>M`N``a!5zCrOYjp<4@Y8#IES?cFhNyg3Te|Y|tan5|V7bajoHYGfTVp zGtk=Sow>%RIyZqAWCYfIG@oQUq1bALR`u3+cm=_|&=mYF>_EZ>|VwwVV+ojXP=6!Wchc>26Fp;Bc0D zi6;9QYgInmN#A2zl;pGcW>)vyek2YQYw`8UKR(9#)vI?-{TLu1 z)wHA#YzSKdr!)hk5&^wkh_Frjxm|3J2DTKzHyLW zljv(+{@0@}srDKIik&%eCYR>w$L+}>qBUOZ=5{m3?ziJl(bz;sSi2*O71 zixu7NUJ&@~x6J@i#0}!*&;>ob0P_6!R1Q-aKF=~Nzj~fO)!h2R1aQ;i`0UYL9Z@mn z66N>I!#~$8Cdb&ORE9l-3rI4Ej;k(9J8$_bx{P9FB7+-HFn|c+DlR@g#rDN$Y-z6? zmglEqhKBOhgZIxfDb3IFSo})uFV|V~x9fZeebN2vhR@7|b=0ww+WS5A=?6T_MQQI% zb1;B;a?;GCt-wgSHJdjdZ0Ec7WxZ9g#af7cvA;{hF3DtpNM=4zZah2a0_G(XVh8gQ zQFL)m@%DtkPY_NV*(OuZx^LYty0KyQg1h7dN;j`_B+^m)?nn z7oj3XqrXWayk-(+r4OKg8(n67?Q43U)rG4IdYqoD(=wPG9zQu+`RDusDB4Wv5m-BF z-2SNIyS_*_t^c&iw<-mZ4L0tl_C%+1jwN1etYf4%+o+;q`d6%QGkDiwEiDg(v5f$m zU%q;^%gIR!tQm}QVjF0$SYTIwb_1#jkLS9t3WLl4kGK2!%dpps!mz(SrW|wNiMNS+ zn|FK6#NwCfoO+&B#{k#*_nI*8B^x}aBMegp;fdRLHnE&HNLwG$74 z^TnU{cI0$Kks-H}GrFP5J3Jep#4OVGiQYm+h$RcImkq2MQ|pgIWtq^?k))w=I<)A{ zD%r@2-H{a-(DdrxI3~mc&t-i7?5lKj*P_KRA)xKCC3Q_okTlUu&1n8Ai$4DvJ&#OL zDz6Mcod=$BIt4lfB)~^sXb`%pkkum&5cySv1iFTj8sT=mhg?2Vr8y8V!8n0cXa^)s zfSv5TqKf~i5)1b&E`Ge7@8zx)2;%R|KMI|$*}8S&%>b@{&HI#D%wEkFn*iwW_g&Ov zVWVZu2(H#MG%#svYo|1Oh)?Rfp{7(KIlKy7W*4HKiFvgzkZyLd z2oP%f5}{lKt9_Umq6@D_l*m_}G;R6ct1+C&+J1|)wXvRhx+ws~Mv$H;!09j{XgT=a zW?wH=p`7~dv#$l;`5z9*B>_Qy*l?7-_kYq)gID#3V~8^7mr2Wn{JGjZ;%8Q z{^)4PKA%~zWaczN8ALlL$?Z|FB)$6tM*yYx^Mf9ju(N-hqsj*)ghz%&bfj-JXF@L> z*&x*9GUYrDl$s!Og)kJxsE~PAe={h(?&5?^PCA5vAmBHMzpj`_PP(tLRx`=mPmCr# z(ay)Mf=W1q)K&wiAS&kAQP1-eqYb1R3$3?A+YN>9o$Y+rqdxx22*uT~GF@fl+>dOD zj5_jNwQc?S*<>0c2=`{EB_^#wygMS?+=7CFk0#Eu7ZUN{UIO`L3yPHSA0XhR}xWE$3N{YF+);T2~N2X5j&EiZRex{uQ>Z+q`)`F54r| zij2sQ;3yJ~WsFv)>-iu3fka~vGGwk0b-$W_ZaX6Tw@7O3Ctysjcc<$XEP~S0a>^5j z?_b5_{Ll7(UV}A2>6O-WWo74&%=hyr`$GSD{m|BkqBbDF_%?p$OP5gYHYwAkTMM!- z6l8UuB4S%lrewj=>0pWxs!6+0&nBYz7!S4-#QZ!WagNhOWe=&l*PTIj3?IKk0@Hx8 zmJ29kB1dtcbkECPpI?HO|1jz13+Dq_0#xG5k&u|k1V#gbRqBSVTNz*jVB!RF{|PXB zPowp%)uJ`cLaDHcbK)EE0g!D(Yv#;dJ5fF+z$m(;iFg(o9q3|uz6a&x$nec+ly&>v zo}|?Qs43z+KR#Gx>sh$sm!S4PtR+xU28I_1%$!`wk&!L{ImH|7Ck$66-o5*Z%(lidxdM7APcekDJAmgUdrSb&>)BCBn5VG8gyYVSu&!i82AkR%H(eXPxk{=T5C2$$k z{GuR~3YD90a-0@;B#f$yOKG{Uo}8ai=63tvuxdEyBCyX_a0WkOjfKzG-M)pC9Ye7f zLf(2e_PSlN64&|(hSo7K^HD(hxLa}Q9aIbpQ0nM~#N~3xjJe>Gqf^5fyw8)}wW*Um z@MJC`fb12cK#K263k$7|LN-BYTtq$)loi9I595B93y`6!^HNlpCD>?te8TuQfd+MIJ z9dB`fa2tp=APRkUB;s29 znihB)piXZBPkq5jJ*joICW|*kyAJel!QnQ{*XZXiu# z(6WXX0YXHfpkjlseQIf$MuE5n6vOf^b%8N}(Hp=lA-@h((MLKPAsGI=34|KuI!J$x1wn$^ zImnp{>N&_!TZ@e#0`|TC#f!Z+&U{eR3<(Y{4=?m-6_k{$`#5qqofleVrt01yAny$lC5GWEW?{S#F~b8+{{do-3B4jnPbd#ST^xQ7?wQlhIan01syc|EAkAT38i&(B^hYr!Ak@&e8<*k*UhW-~ zfy6wq9b_~ss1yZW;I!m@v?|+Hw{uWXelYxdN*g1`+jNPz2FNCo9o)K~V~Bn{b|?kNSR##=d`RZtWOIm8;Vb_75YsH%@gW z#Ki>Zi!p>TU(04qONlJcMfGSG43L1GBdrUhlqoM?6ewp4DcPg( zdh?bpUAlPLGG`QqYWUPg+p7~c;QlTEbqzO);MNpA^V%JxPm*H!7k7kz_CnUn__Wi) zoHiKlIkLM#FG2-$(ybaPPY#dPb}}hsaQuv|lYW|{hMd5@mS^TBij<1}H|B_LAX>W{ zWiiqP`b*5?0F=P~6D%j3i0_plhmnhOkrCMpA1ER(GAQjHW9L3h>lA?ciyqhq|=F&L`907S9mvg|$wl!vtYFwaP2eA>1V<&ih(TYmj+aThjUjbZpxFyPk?6H<@mS@f+(8j=_n@u5Q z8e|zU3EAlB0QZPOL4D-uy%h(GryO1`3q?WF?u^O;U?!4iUNa_e1%+@>=P+`-fOYJ} zSeNVO=Wxna^YAQ$_!3&A^MHn5LNt_nv#I_zq3OW|M)VIN3?);XDG{Nz>WPMj{mr_w zC!A_QEudTo4X~dNR>BI;Y`_7?8K(n#Ij+&{Lf$e=)Gv6^gH5bHJ$KYsl#ONlmKb>O zp|q;9Ux}M}t5l3d&08aTw*C7~=lqHLEi?CbB$}tGbYxdKhjUM}U?xc`+9mA;B zTqI>D=Is68fQZdGAz?d2)Zc|FaE>)hjB}0*_b_5$3NiK==7GxFUTbSXqRV8%lR4N? z)qHUdM-D3Vgq%Vi>977Pvl?Mo+{TXB?ETi(iO$)%pMMX}{0nI^TC&g;$0(yOpADuu zsr;eWLa-HP{2ZAX=HVkZ_qh2X9*$pQa0x~PJXkf<-eaiSy^|I6{w%Pn;h+TcpG6X14W~erL5l^{(li%UchxnfLf5 z-L<_;vzf)T6a87mqLsU|qMbEzYf|(tkrMX^GFgZftO0vB9C=Us+lS7$6uzJB zLEI?2%IOq_dTu+Es|6VL2>TYXugP7wSvjn+ZE@q3umu#Byq*R39`Z^wqo!{R0Of9F zv}SrTfg&~>_{0kKGOM7W7d+4HXeWgH6O7xox@JL)>IdL*=fR=qnD#=cKwkwW^+(hrQQNs1^ibCW&s? zS9e=V0#BgPOU=+Q3;36gSY`?1f$0{z ze{I98H;O9*np#^!0q8^=OE`s&6CViVKL+*TajJ(l^XowAY#@cJIXS&G&#o}fnX)aT zT$Y$1z~KME5yaubQweP-G@m;CTnABY*o~Fj?8O*pEdN|?=rOe$A|TwX5^Mx(#c$xfM>n1Y|+u2vNnQ?g))4Dp%07A7yC^kO+`!fIB z#S1L$&l8ArE<4aNP}|?aRt*4PE#zY;IPF^$0^bz@@E6B3aQciHlFU4QSEjs?@UAtR z_JkVOmCiL8=&x z8rbIb>-X>4wTnF5{1scjv@LGZTV?mtSR9|cTnjClOXs=1ffJB>T7+&YL2w4Rxwul+ ztW8me8{pFaBFlDk4!Iy?XK-6+-XNI@tkIq}<#mLXF+B-DFn#Fb%C+<$a+dL_aIkNr zGa67E#?g#Bc~FyAd^KbL#9Hq%Z$3>H9TJl4(=!(T;|*$|tv1q)0`wpE&HuqYiL>!5 zc+_v7VjDHu$0E9;)9mJUy-)F4PdNUup_E$BDJvAd*{%{50w zZ;b!o!pY+Bm|1`M*G5nJFw$I0ZaPkQIWkJw=)(VenfQ6G7)=>&-Bv2!bp}y_F}in; zj9zX-57{G_jz6MHRD{ zLvt@&wVdLkLT6mC(af}0!R;QjX6_aE*KZ=drlUpS;g>I8WMjpRO1UHl-tOVOHoqI1 z1%DiOap#Yyue=IqE%a7^Ei6{%1iV&?RaMIPfuKi!i!kjfLOdBd?)IlYkMDLdyY49L z`!RMg>$lIZEKqs0_c}xDIS6n;?B9adZBb1AlsEK!u<>|ul&P?9^I0_A3Yt_#$qx2` zMj8&*^sAAZ_a405bnwpRmY*TZ-&Srrn4AxQc^Sb8MjVZM1ale;KJ!MHQ6S)Q`~eE> z$l%P~m|~{d>dZM;0s{kCMa;KSBj89vssM(?WAsNGO$uECbtzW4Mt#o`;Mtp}yen~B z_b&4>?7<|M=q$90G9H*^_g@#2X`|v!oSnk2BX>5d92DNg?=(hRzwxve4n0)~QqNb{ zT@;ygyg!ZAf)aj)+ySfo!8R_lG9Vi_CA@l$;&+*bDTmcUYnSBQiY=NzQ`0!!(q8VBb^j^^e z7ERN-_RpKuN^U%8!2+4}Y!(^MX69*n{AuZGr3moTM3X?0iQWVGrluzHau<%g`fZKL zC8Z2E|7pUWDkyU0C7p^ca+1D@Q0?5$DDAg@>2*V!mVVn;dHMi^I64rv+nwHPx#7hjSy5fs@2LyIETAWIw~FUPR>o`ZZ9Ojx8_oYTEx9viNkG5(1$u@T9R52no7FhQ z2u=Rvi7badmndTo6cx$+LPoGjKtKsGlPU;q+*BaY-r3pN&?4<>+Ss)9zux&Xj3qr7 zWudcwK6(^pzGp(>*GJqg2GL*(EfYWGQIP#(BocMphY%x^0bH*P-m>J9&3@PZ+!F^0 zc1lzSQcp44&U>&K-v2snqGvw4A!rVQ{__`n0)%%i+QEiuPfqS)ad`+&{{_)s6-b4W zi+TK}#v=B!_i1SOp~dS?`&XE6D*aSRfZ0e>UX=8wRS4RK|1@2qh#ggFv=lBhD9FJ6 zs({QZbpFSfUKhsihN1p$40upIUhjGUgw4B95d?HTwD-N`^glP+s%>T-Lc9v#7}hmN zM#T4wv$(ABExKD2*2sgl6Nx{ZIxQc9Xi55oD`1ut(99J+GSJqXVw&T>jESxstu8bU zL5%i>>}-;*AnL>HVXf14?=6>3dEf8c0iB_?;>~kJ4>>qw;0j8z&(4?(;QxwG@{wfd z{fvR(9=GLI3I4!rQ)R3u59U`UO?&?M&qyTyv4P^uIddY_lZ@p%EYL77cK%03*@&}2 z^){%LDsfMVPy0~(h(K?OG2~4`Dz_*;xBwgW#&Ki!=|4{v39hiMq=+ov5erQr`7Zc$ z00>ij`jX|#A0t0BFsh5!FGTRIBJVM``^6M1?f-yMs)T2m^7~|j31?uGrlj!t_3LxJ zFJH3z_~#w>u5*5b#f{L)Ogl0Pt(6FPlh5T+O`@Pl9+Z*ko#L*z58i$Y1ZjI_fHw08 z;%KI&OFv!;Yvi5slvrgDPbKxVzQWbnE8)xNIJXk3=4H(LM>tF?M!bzdgzMO)9F79YtpGQK zyRJAdHv2~eGS!NAl>wn}G$yF1aa>qvNpOBA^WU zbHO*o^b#4UM>h{#_zsNwq5#R`ki01eQ>6m`i8X|Ts1cx=ufxMtXw?vc6jhQ%gBBJ; z4d)NZ{knzhV#?=+Isw=eac!y|G_|r5clh?CInC|@`#+CiJ{FaUgCn462og`KfM^Jw zfDKCJ6G)jcC9dgQs(=N$S#+ulC5IrwEUJ#j$WKEYWP9Z0Nth;_iOF6BmWps}De_gr zLSia zF026gMS^{H{8W2w%Q&AppMgR4%vlupw^sxL5)4t`9A?VpBR9OOE?o`Avl0eYIZ~CU z;OoR__QVl%7ztud0{n;qE}bfFtlX%N=$R0LeSRJ~cza`0`M0_?05$)N`nq{he|ZNR zh_p{i;p!#RDYh5+`O#{)1QLKyOTkX2%gM{oaor$`6$w34dH9af4S^2y_L=)2%BB4H z;BxPRPV@hYpzIkX2fW%k0CQ9k&scr{$52N{#~RSDCgf}@toXVY{plWaXI{C%HEwzL zLp3At|IG-2p|KJsJuz0M`HJyu9l22qT}~6@78+)C`BDW2qbiy z_VQN=7IUkonm&V2#k-e*?V;4wffki2ETJGK9q~Y@gCwnliWGe`ZZ}EMF<>wJ{rmT(dFN2` z6++c{k2LZ)G2{lib*ON4AjR}(#la{CGPJ5h!C&k#YS7#@s-;I>LI1=CI#>n$D7SEn zAO8&C9xRYEct_Ud%UftpXp7Su z70gVNu==>yHBB~6p)9fxpCYvKv18zej?yfJo~T2kQb|=U{-fnu-b0Q)TAJ(YLQ%q` zCX*i>@E9CuCtR3gn)PR+LRIk-#<7K{*K*f{4y7e>cG&%TA*U$uNZwl<9c>jY5E)P^ z*Hhv7&fxEx&L8emtCd4@x+|EW8Oo$b(23~ZJ;UEJnlU@0*8o;E&WO=|TukU9dJAHaYSK!k8O9e{dl)X&GL#_Lg0eNu%er9V;C zw&{HTF76F%5yG=dFz&)o%-T`pG&0l`iNpB1r00Aa>RP@8yu=V&4MIf)&^U;@p4OR} za%r4#zuyPXVAM2htK?*!58^K&D6+gPxO#j;(|Ph=C~9m%OMltJoEyO6527h=^WjIA zL-)kphNn6Q=+4Vgql87Uawn;`{RLX^{9KiTa~EP{ zR)|>>vCuobn(Cm5DkDZ%&c+rAi)IMTMdSL6RCLu!mRb1a6%=&9*=>eAL&W*pZ16UO zR7oXJn}CBfV>--y2det8Sx@jAnGDg*QHLa|v-=SJQbPx>`d$2u(O9WyzAunPxlSEz zKSxHB(`GGLVF)`xLfp_jua-z0jvSNv34i~Djyoo_-Fn!u6<6+{aIS?i(c!&&FC&eJ zh8K)H`RtNMi-P3fZL+q&Y2hoQOEK`a9HRAlj*uS>H?590GDiw7 zSLYZlV`RzOw);0mH;KWQAl(v#_{;i@-8z3dsOESReKfLZ!ll?42O{T5H<3yq^RuiR zKZE(P!PoDJ31Tn8_#~@^LZFQ8_-D5W6e+RS+*BN-*YYpP!6JWB+!24EF!C7x7fmX` zKf{q9$#<142Z|1~!GWrT@R7rz_nC5yZF@Pjw98iqV=+a`YJeM`k#*wAY-*`%EcRV7 zW2w!wGk@WyG*x(7ARUj%DMLw58G;5Vw3m&sh}%Ym=I$5%(_k<3rk%bBqX@F>A^PoZ zZXbldt+Wi};9JeM@$(<1f9mM)eLN_ccPioRp8&BR>;Mk_Vch0BT>76??LKvN@Bi`d z`bEC}GA5F=UYtHl`N7U-F+!HTVx63MZ*${{ZI5cx}R+7FQihi01T1cml=hfVu!h@Z8z@iprI zvqrO>z`n@ti|+C7fqAj^JM8y?d*mjxT90G{6U&T9d< zVh8Pv&)eNVX~q~Q#ns)t0#d9=7(IArZTu21!J+x0{@%8j{|CYQq!@MZV*uT?v=Eh7Hq6qynh4 zwe%Y~a44y6YJe!J z#At{TYy#QikoAtY+7$F+2?i+!APng8nuU|kcI*hY`{BAS5=;^&s zMtY0J@<<3}VU&^_viZT211}f-vArT`k?S`y?~yVK~V0#02vf;1obAE#-6q(5~om; zB3BY7To6=P!sm(W1nRAXV}SHWF+cCFvajH5T-undHls z!Yt_92bxw}dVU%>CI-SV~g*+yYm zMe;kAY+C}k9YIb3b>$Epg>U~;HNl0$T{Q)ht8|)cJ=pCy3kylI29`MWV1MUzac|5@ z*;-bE5xQl#Put6rF~{(JH&pm2ma~1~eM6mJh5-j^N8w@;9N#FqALLyE`g|W52^n4U z>OS_)8u=Rase%RL+_{!D$xWGm`zN)^eelr8n1TNUHc5}H)r}I&hqFv46h3MxGQRtr zPE2;{aXME3Hm{@Fe{b;?(tduK!7NuQON3#Mh(-OIw_Eq+c$m)QT!kQu(+9W2Xj*cZzPL+1<+!^08Gz2L}_$CoYOwInXBa1FI+~0qz=+vR% zRBdL$&g!eJs#3$NwMlt@`X+n3@(d?upy2Rf)nRSbz;s>C`tO+Eco^K+o1UJYU?_#4 z+@)_ExW4Xc{SyVr!`{U)Wu4u*K(A7va28&pBO-z zPkUTfTbgx?)If5Yb;ZQ4A>Eo&3E?|0PG^xy&TN`ktu~gG0m7uN!?qk2rWFPgv(?Zq z&^#kE{r!ZYW#7krPt8}>yZ;?u+aSJ{->IsR?Nryd1CfwXvRs-3V6EJmCskoZrwV(y z{Ko@US7UNgNssQ0Csit7u%Nyx4r_DT^!UtzAgs+$nGyP8D1_PqPPdvGdWL%u10s2No$yDzRn%gAZdhiWPkv=Px z0wnlpmQ9x;(;zZWxsZMk(Fc3gZVy-2g9&CRV>VlXvvsI>I=(59hfov$rkuWu_^Ja( zr_Y+%E*jl&cX*Dg(gAf}7pwsgQ`(mTI~K!0u6Hjp0vtl&N1^;?*+}z#Nqq4_tL0tm zA>2SE6lE&FNVG;%<8mJL^29p$P2~IKw8o2bYFU0>E?)mh2op}VY-IZ+ZP93)Jk|wq z$?ua2WwuY6lSPDh$Hz2qF|NDgrw>+cRePxz!uuSX{JI9_fxIsrVP2O4zh! z%`qLKG`a+~3B0I6w?1G+@FN5RhKD+cI8gT8srrnw?I`e|-VOs3 zf{5qrY9+ix&Y*_au6OAC8tiFkFmeP}a#TIrDN|o61K*_7Lwc;B%|NiGgq~c=JORK~ z8KRQd?odL*Ol=J~vsD~qK~`bYFqCc$8*?tA%OGRUx=CAG_!u#>W?CXC(6<*1O8a>8I+*#Y!8RTB3KFrxz=rV@}x1wPhztMo*lbr^o_!l6{_ zOV6vn@ev0{yBk?dPxisJ2?N0pRYN~gu120n3Kf`M5J<*3kmU_nv;;qHDGFbS{w#R_ zIJU-tXnrb3`$Z>eb}R(4%$^vxAP6)rwHm6rC=ztEJ!zaqfFhhc!q4lf&pZyDY&8U_ zfP;~aGHK%zvREJqw#BvGFowjpK?ySqDnnafTwuB9FI-5% zI3dHgs|jf|V9_#Oue!KgL;=Kmk?O4&t@i=i3u9sNx4?1}KaO9hh|@hVYxeB4Zn;m3 z{kXd%5d4*+gAB2J{1q$5;sP4G+p{yg`IVvzn)Pd=HGmth76L(C30-(e1|=K?CJtV4 z0f4O}A@!yHE3W1&Y@d;79m56`9m(BFYZI5{m}( zH@&o%50r{7;5a9Xbu_L+!^(`Y%xbCLS z*(d)jMan=oW6^bfntc9yF$3)kw{)LrDX$Pzx}zRd1!b!3fVZ{FBUB2#^kf78BTBM! z)OI51@faIM>GB#>>fe1nF)`vDRD^)1j*}}RAIH!A?OVkIk}?8-=324s`(gF-5f`15 z5jfpiYzEG>c=cB-9T|P+!j*d}fd#5@!-~O{>kZu4NhX5HcC0v8!b3d`fjZyuNbca^ zu+ZJ`Orqb}KS|)nMWF5dv0Q4nr=f)ZoVT`y#X`5LXUr0BoT_lp!}#L6D(vb>=ojaC1p0pL)9GV4Nx7wg^PNTy1EwQ-Xm@&vo`Dl8pod z!kW^=m}oEmiP00dL$A=$(NPGLz3-!=FcFSlUu_7OyG;IWUEChTw@`%(!G|=#I!9nT zU4;o#T0rlbE7PX6Xv#6T5}=DvN&^SQ&NvVRQk!<}phkPQj_50WX*T?`OH!GSx2|&Y zzPSt$%hP!b4CT;5495M9RD8HQV9^|{kKlGFWjTD8_5S|t+n4Jb0=%q8Kiq>^vNka6 zT?$S~72@e|e15U^X_W~m78W$4AagHy-l9deV@(6FI@X6YETcO;*ojCRyR?%AYQ?44 zA}`}~942!70HN60KA*MK*(x)MPwVOW{68*+eKMq_7AV; zyclo)IA?8Uf<|!>vR^U7f+rIBgOxe7x-@3zRtLA}GmG*IU(Va%Jl#C^Dl=D0M@J;U z7Ku7W3E;1`62-nEv5~o#yjfti2s;|}teuEYz9Rh~z=jyd1hSYAQY0Yd1F8~&x4NY( zHMWTBUPXGo5d(thN*c*zJUi@FiM-*3$NVu*XXkfR=HW!?{yI@P2-#v!O_V6n@j2U5 zcQ)F>PqIRoo`g+EDPEQ}<>ZnSgYc5+f+z?O40xX}ut*|BM-K+do+czk3k>$=-FPI# z4RX8>(V%f~)_f*Ks0m~Fc#&7^f~jcnX`^`IJfB&S-&mRRo^@%nK8+EPB8?}PFA>9N zs7mlf%CTCKj0a}gk?70@bT&t%4PX13vbRL`4-QtqH#C`aS?+RYm$<2x*c~bO@-0a%Fwp!|6Gi*Zb^? zpEK9mGatw0$(V(JDWeoi6@!Bwj;2cown80ZodN72hmyT`@#5qS#}x#`bsZgt%IUUd z(I>LC;0~>~o=4Y0IlQfuuEgI`c? zN3;+?oVPiJfx?pWdzIGkYq<Fc z=nx$MOEd;Mr%MP^QfhE^j^XSu{yIDN9#9%p(3c-X(ma(2<#&LGlv6CVVW&k>&-RCq zV+v7i!v7*Q64yY)HDVvYu@kM1N6N&wpHU6<7!p=+Q&GeuncNYx8c+uAbm*aY_}YE0J2)`cAayer<>eCQ*w(FX&!Vr!33W1W%RO#+eFH^H7nG>> zy7Z^uZae@ePg3`QMf1$9M+S^g5tE^V@KI!4Q!}T~7U1G$FjE(-bq>WLl?O3{9&CL^ z0{}=tGa{69zRk3B5T-;qHb^?wxoIFYfbFsm zFH(Uk6ohyw0tAvxA5bseg%d;o1sexnh9pU@5TyVkNNcv zKzkxh4+M7v@c)|6P1)l0-GL7&1ZGyd*1MbVcF1d!+XZYVJQ=eKtR2?z2@2j$n?_lN zCE)$X5)d2WsgCI9t>1aFaiRq;@#yq33zdXi#JlBKV zLjx?qJ;L;co)yyQx0u#Ls5$;Stez)!fudTwik)I1fayKNxdCn=WgBo@ygt&W69g9* zi<=7Dy3^H!E7u;=tsW6L6Wu)RHBVy^_=Ms>ir>XSYeO5-9WKJ}@nAFrK*$_B16yJ8 z9@kIg00gKxH%ka{Q~)F5b8!Fs_VX@Wid4wGT^ zm|YO#M*&3Mke$9umyE#vJTB!g=eMap?frnh30T zNTm@9UErMc_*`gAvLEh_M*b60?b5(jX@&IqFK`wy+w#A`SrWSc1ZNGTcD5`oW)(*^ zWr9?41R|n|=O>(iDINnQT^0Uu0FNIAPzF>08=X%(i}O?JS(#&xv*}uzoHTVD*3EG9 zy8s{Y>rbPt>_w8$PX2kSd+GPzQ`mndxa1o-UAU%y{W1Ru;Y|LgrT9-Ahy2Hd8vaR? zY4RJj9{+laNla++8~@keLIiD!UO-;@wxjS5k{>?R>IKZrGbmn;P63O6>pkOR7>5lY z-GfPITIE6H_<|^g5G&9s4Y*zxLusV`>!_BQ6M3xxzgM@*0$mb!*Dbq$27rXS&zU)2isgT!WRSys+Kog+sIlL%KBrg# zjTGyuO~4qm9QyXDU7cOKyff6P3f{Crm@1utMnsVc+abnnej^Kk#5(g!c}cqUO{0nP z_RU;f0LfaiSzJ_vLnXcTq|idr3$-h=YK0Jl#E_H}IdXV%-1&_+OMmp11|47|aLQyF z`UfW1}!0p&u&Wz=x7fG6FS$M63wsMXHn)=&&S7 zI#m3X+`BOB>rk9K*v?`@udrPeVDABo5ZpK|%0TsHLC+ z0sY`6u{QGRQiDLpyQaQdRHGCjBS9`OP(iZWc$9ar(d7O2?ce_zfUia7;9b}UXC}@I zNV7xX9jNqXUqf`95a5ikIZ+erz_bTdK#Al^7f{XvC_#*77c$h3XDP>wBD30*!=zG# z0uvCEmY$wiU{Z!yGo*Ch)}zj?xc0ZYq(?`S7(aoxYJ~j~s}?3P9*&V{UUEW0Al>{t zU~qza2iJ%K?kq(zqw>^Tv!Lzm`5)gyQ3G2?B^)`jr+}KO!be|J%lqwkh>Od0u7rf= zuaaR$xyZJ#G&iIw_+ghQp4%y_=NNrR3}5rAuMF9aF6t}Q-NVRt1jfe37^wTPC~gRj zHyn3D8mHUa_{^ir7Fd2o7RJhS8;YF%^pdG?*|KHsFo|LxFrZ{p(R;A;3}pmhiNMuF zk;UUZ8eg57u>r5{LaV3_dQ#cC=P)>tdgk1aEuy*oz(3q!Us(Z$dbHpl`{5xOsQIWI zm30=rLHR^qMgaV1*)w*a3kCrqa^YGhC|I883BIuCM-423OLE7X@!Nt0fV(^aIG@Cj z%#HifQC4C@*La*XT69f51a2R?zI`it8TztFD@}YCS&ckqY3}@Q9Ri7(`QO@zDJ_E+ z^$LO@S;b3S#&||Px*I|rR*XDJ?gN#M4cWx~_I|AWeq4@))ct_PtPp^d*%BBD=dS(z zXCy3gHIIjz^7t5>SWj+97rQWNm7Zp{2^i;e<&*0{*;bee3t_%e93fPBe~lusLp#dR7VRJ*hF zm_T*MoJFhN66*lEg(68*Kr1KpG)!mWu9K4f{%ILuQP4@ta)|vNKBUAX*$RNf-9#nA z(f9!=L`ghAkEo^K-7_s)ru*S9M0Ec0&B1^YUJKfereVDH?uaV(BqZ%9KQ@2IIrDkA z10CDO_L8-DB$A<%x87dvSqFwUdlkzNI&70P(Suf=Vx>z;n@7e&^Y4t3MjZ45DA;^Q zPghflat@vbGb^9y6-p*4yy>h>C3H{URy_NTWx8Xxow zmt%F6VP=^Mx50{#=4n9uqyR_=)8JW@WfAfXIG_dnAT))hcA-Zv3^*4(_~px&TTg5N zPvWD8Szdnr`WtJ)K_(#m`cT9P1(fK&b_jL)Q#*!WHpDqdO%O-BEgzZvp|YwOGiD&0 z+ysm_q~?GUIkwGx_|;o=QC;%q$NH|K(0wcXM^TEsv?OfLrqj2SuP^0ZCGu$dj|sW2 z^sg_ZsB)!d{Y~(fDR{n#ykcf)N4LK19=Os92oQIA+!nFW%7oyA7~CGI0KgDN06g22 z3VH_ZW7p=|u7GI4JYc3u>$(NiTL8|NZu{}H$*49iKe;H_nDc2uy70ltqm9*1#D`A!nVs18IAkh~IgGIGq zDe-YPf{S_#3rkoLu&P(}IQ0lZO+d9B>~Oqj``o~u60+XNSG=mkjZnW}&Tn#&sQyEm zjY)h9Dc>M-068N_&-p|;Y8YJSwBTStB33>RS9vSSQBP0Lrw=}V+ZKy0nh2pzGkj}q zZYEbD$-GV-Hv@H>NCF*^MBnUOOuYi`aa+D3*gXZ!;GKjV?e2EeS>G1kK*&~5TF^+7 zMBJUom!Iv3`} z_|i|srw?CeiIGEHEMqw?kFHxFKU{i_BTVGUWAI~T!193)^AO;nw5u%XmJ=y#-jd8ur(6g60Q?qcEUme$S52Z(2h<#iD z9gKS7sv&z<3DG*jIN;GpQAEGRP3v=Sf56)s<48b{L(Zq_yqss`@U8LBc1Be7?FjBf z%$kcGbzB&?VS&wam>bUV=PO*ty&&+3=o@T3BCPiePf>bSMBSdfq$lHM!1#bk`hfgk zW;L&dIYNt&6a- zcQzo8mxl?n(w&)qw~MwO%^#t(7*S)~Q5}yX8v&c+(K=)iAG{t#W<>61BrISfT$*Lv ztKA1m)2ssO{NqCHf<&mKU6SN8M6Uz}=h#BgLi5^`hsdb(L=q20;IfdcsRAdizy$p7 zqDo=S%s_CMOS`ri6-o5w%y@P8tWIDpq5sWiWVDlL&Bgv(Zpi;Ok;_azY^W#J(&E|7 zSr=BdBKpDq)Ld#duxnJJuvaaBvasZ6XM#s7I8Sm3u|yE4Z(%gfiFg*v9d>{wN5CKt zu4%CRPEt1%I8Ykf33x}|MiReD8qWG00K&GJo8Ug%B`Bh&hhzo;?!d_HcrPMAz&b7CBsom?Plx93tJeqrKu(!s8wO4aY{g z#OWLg0Emj-isZo5_+97!F`;7sd(*%|c7Xs-FK?oe`u;`5&@2g^q6OC#{@CRWD z2@M2}cW!_8eJNT4B){y6Uj$F-vW3{cn)}L`lCR+M+Dq7m~??qOB#5wOo8-z zsb@K)f@MrnS@F}2Z^rdRbn_VsL_Xq{gcArKtu}{Io>N9^6!ocv^;5(WR3&6si4NaS zlae=Y&4&9#9;e_j!{=iEj`vPTFjSyyLwy?S34%x4>g0VdqvT za-&>`#JBUH|9qd{;hSX;o@X)_!NRei6~+Z^>4$;C2O(zJ4_wn7>%odL2ut@UEE1Xb zrJZP8@p>cBX(-!TFl{~hHVlYmA3!5Ea{e^8K{>oyB>RmM9?bi5GK&2@hL9rmc+ej%x^rJqQ+{?pz~r=m3`F>piUJm&_m=ugr9_HQT3OQ z81(ySViH)ol$%9-47p{y@YfxMaKunr4@Vx|j7>Py^vQV$6CM*%a3QpQ0?^$JUqOl(WDY^-i?{gX0HhTNDiGHRUj7Ii7B^KvqVFc9Q9zSyst=*B0!BMy${^f|JB{O z|K*(LecbNcyY8`!on>tyF(N69A}PtNnYe`LAPs8hFvFieFjG@WNhI?ITX zQn|;NB*axpr7}Y2a!9FMbwA&k{Uh%CHxF`keLvsN;r)J{K9c2y1S_hwHIv4h5seRw za{@nuq}5Z1PJu1zBQ<;wAb+axuPP)FR{dTG8-UFMHzo^@`cI5FbbaU1m(GGMhZW;&0 zoCWW<^Cq&4goWSyV+~}cP7>vH;bdK%M!sVyKX3lwC_ZoAe%YCJWQB&MH8n$UbZT|F%_c^}c+oLrpYwG(tyfwD|hkD=`3M1Z+uF=#S>32ob z1ppo?<4=|6!#NgtuKzmz&OVpFQrB5kYqydP%o zJ}IpcttO8~15UluRyzGsPy#42s9fW9^C!8NUam+lJRJA8zYXJ$$7&ZhDmOzi1%yN;JAc96g9?;swS6#UVVs1?X|3)&IKKJ}BJE*~&JI`PGD8s|8%W8{IIvX*(Tfl0x$!Au3peTsXx1bPCqGnw! z8b?NSY!NV9^p8hbv5i81Qb3)cRsK6+O@Qg8Th8UW?+FyVHsL{hK)!Ip;=2|E4t$$r zgGG!_@m=+2oo&?v?gel)33K8b+vk1dx2x@;BLbEkQNbGsL$TEU>iO5OP3HjgOq<%1 z5@J+T{>rvY=Fh_eMy&oDUoUscPGfqyCAQ`}97t+^&&@7nU;A!9LyWs z@TUGbiR7{N%6&;isy|`ECj)v)K3!fcd(oY{K4od^QgA-3R!T)K$wAl?zDLz3;7cw6 zToHgALQpX99^Go{pk?NJI_@$$iG0HWHkb~CXCSLQXBGEl-^KK2hn5brmSk$#gB;1? z6w9EL>hSbEY1D**vcqWTA=;4onkb;Ne4ot}b)x6|Dj*=VRa-IOlJ=^zRHz6&J|n6r ziS-m^Kgt+k^B6bPrP{qI5EzNzY>s>Le~OXja~!&h&$-c{1S;-FX8lxdRsxzaG}r&x zsh;-uG}TZXP2#*Y1rg!AxD;M3HzOXC+ZA(x5o8M%+dII`W*Xx?+;#aHW^+=GuO&KB ztpaPX?Mc;n>v!>c#2&n>y=EOM9dSt^i#f3&hraVZ)_re)I#DIP_vfL<)@xMe2j;uq ziAtg?Xyjh_yOMiBU29iIBa$2A-!@m8c!^dJkF-P^5Q=tLEJkXWn8J|mpzUS;R*M&} zpz@yW-LL|4a{~`wazf8`7K_j`UG16)`cixfqy+k9XW8^ER; z$0kT$!ib8?61&R+=YZEI_u_$a4ILvc&SbkJWlL8*1lfI#764Nj`C~dVtq47}e$mH8 z#Q`+9{<7(aLX~cFsOjcsIxd#Q?ygGr7C!}i*{fHVcK^ceWbfb_p zB?(9ObTmkCirjN9*fi3njtY<1GZDJ}V_|MU@XJN6EV@ZhBT+5MCROg_zYJlc9VV90 zvpFw4=cQ4d^Yh-F_T9M9WBL#+B zA@4$xAbjSAbYnnWV>IIv3Z*I8G?p%BM~C!TbyvTB(jCIb1KoW|uHcbv$F7cemjozL3D3m3?CeG4H`3N`t_G!7^Y?LeAyq48g=T+Zcb*c zA~u7_XY&_ZyVq8601Q=#4EY78e_TU5d(id{xqPt1vocWoz`=v@qaE;UtVGkePhhsR}6m%>wc(SDgi;>(1>H{20&azi!N{?q85pq03cJiZEt#CBcC??DSbklLkj&XUKy z=HXk~KM)lPhpCQ&vne1s%qSdfE9&1LDf?VEnIV^2{$kh@F#iVX2c?dbe$4w~n_9_V zsIwU@$vTKEY%gk}8V1b@+v?0GmIXWWA9fbjT%TtNuVzmt` zgWfDIwq`}I9B}P$N|(VRO;z-ebS~M=vwF?YI3j(Vp&i>Fupg=~*2%whw&eOM?G5f;v&T*17vvm~R_2;^@@8q~?$$*wH-5d56iK%9juDPb)HkP|Bt4XJN{;MA; zLiKw5s3^wfe^moHx2gF_$j8H=DxjwaO0A6xn-1g)@2q z;gMa>o#aFyG)KjLGDTtP^>}~H-d<0I3L@*@>|OB<^-_ocm2xe?4^-+#w>>RM55}9f z)a&3sF{6#0NgN=HpI<34L3JCq)b=X`M+Gf6u5P2=--V@GN|5XRr?%D_{aVv6>^J1X ztXihTz7){_E%;z$fO=JYrE_*nhQ&vhBh)rynSK#OF&}vf7*Y=WD>xK2v%YP>CG-fo z%cEBCYL|vF0UhBylyMN6u7)qGM7=o>2`RnB<{T*jaVqIe)>h)kUzZ&A_$TL&p3(P* z6a~dSqlz#L$Yn93JjVwEdgj=7?V}wrXrV;c9TLl#*}##0SGMb{Ppuio&~<5xlyV$` ze&DW363{mQj>Mk6YPrwlu^wD+m(D7jq>va^^3Po=)4*s$tjyHhC${WDv8+%N6O-%B zT%vS0zwh-7g$k-olv|2Tc?sZsDFfekSz8AQQjvLEEmjR5=Y7X~zhO^JOV>QPGuNiI zXug*hd4mSlZF|w-4A`0%2s-1*@v4R8`R5PUd@ixiCH)ig-aP(GfsWAgT3vr!(yT^O zWT}+wFOtYi?)y+z807{YQs<3Q`t`vk%2fsZ+A*t!iE$n#*wzzeO*U_WAyU=5LL>GYkl)< z>t?|ro6_S#4e|+BPp)mb&8lj29INP^yZzBV4vvfHAWOFW_1Ye44oNGInEn@Xj^LF# zxDR4-j$`IJmiB8HAc=$!w<{=P`)VH|ROFHcu{A?dMU8$%H}PB6*E>Gu2}HEe&Hw;#``bI5GMkgDj&H5){f+Xc^1RC-sNT@%KPc_oYn&;H z@jgd2`Gl59{Jd?{ZyOvWHeL6>$%9S=b&hyeJnj$?K~(s*!I#*o^fph~6w=Z)(=gOK z-5-!MX)A!)Rjn(p&f!QJzs9tD=)JxhLDf2Il-Z?26#2&oi=7{?K-zXX*;L+;ZpG8X z$STr!W5UxoHubZeh`N1m_tSq!jvZ}We5;&{jd_GxYt?GZa7=eS_1NlW-(ZE&0Ez5H zBXg?f4CztL*j`kMZ%k`g4K*woevp;l|fJvxnSYv}L&dgnNrGosB)ZOg;LHdXc5IRBKZ7 z;W*q(In+Z;rH)9-N@26L!dBi)Wn*ffPAv!QCVKHlqyf%?*L1cv3W$&Rn2}|&VDpGJ zKfW?4U_{rQzR<06i~?NawO3m)6AjhVcIVJBNii*jJv@gu$uI?Oe%t_~1 zW!bjPVU|W~V#+k%PYN)z$Y?dK%PPKM9L>~Y{M-tsPdgEjx>Ti8EkRj^Zx8wOp2@!D z`b_WmeU45%&OhM$NfNc!8caPiUTbZ`0tIr!)hq$@rpv?jX z^Q|&2IWkede;X2GX^j`j0kputkYwNC%8j<8QO2>@%Sa0I;8~<9-Hsfsbecjwe zmW=&{(sz#pY93NFxnf4S5q5G6$jSV_=@ss(-anGIZ+JViv#tca1E<{1N}p)2hyI~) z?X3t2eAPZSRa?cv+17eI+C&s) zw2QW>`e*vb4RB`N-J1wNtq9U!v-C$`FlhWCefy7D=cz|Jb1Q5TEHhPkbJjGe3z8G< zK55TQEr}M%Annl-r2tPPM2SOy1z~@x$Es)1J!+wd%(-yk^is#%EvR1|XrJxDPs!)C zxc*Z793<$Z^*z5fJU#%Ya}$Gd7a&1(<+ta8b2Lq|Kxt{FP;wr}U}>1;;sjOJ*v$2d zVUp0HPysIKT3Dq0SA^rA#sr>z@pIqQq7gG!9@*=ZAGVN$a=&X1g)gzT~b)bW(J|3|*Gae9>!(p8KtR=8C% z!QA~L8?Vn~Q{s@G`dK9X-4@gaBH-jbUv@2;y!87Y(5pAltcsNM$l+GBGs83uKSi`= z$CH)34P-05s`X|w4*V>A*yV+i973%^*s4vgkNO>{5Z>}~)bF+paa7Ro=yUr!JF6WV z1GnbRmS+Hdx*`TTfwE#Poc}Sy$qsZ{asmZ91CN)ijL_%|^oR{?{cek8VZ3g*H{i#j zQF#qlnFyFe{;P`5N(Dwq;&#Uf6R!;s?SU-o2t&6cqD&_%(gq~%K5z&@`36L9c*OWk zu0DJK-_tQ*oF7r%@s%u`HPAh}X8s4^i4 zQTiP1$tq^F9TB1!c~%HoMlV~+zcLgS6}Ph|PMm1_cD+esr);taSt5U}B+7(GT*fYu zJ`g-h7@VBMF!Mm$A{OKTZlKt8K(l9ar68`i|4x_3ww*v}3KhRk^>DCz)ax2%${9vK zg$1@4Jfi}2g_L!rh;8ntI@)r|N@lc} ziCE0dG{R(A>~pE6g_n?Rar@O}ur|%^46)i`A$X!BHDSsjfm~Y@RTRZ6gUfQqip;-k zENykZ=PP=V{d>PPqup{X2+9psr)u@~PxBq;ps3qOW~xACa#sN;8>fN*=mN766aJ~+ zOC5_b>0IP1vIbzxADGt?wmSI-WmG0pjfX5SO( zFzDLmNLJJs<2V_|dT`X@_ndP#rX{;-Y1G#I+- zG^fL_cv@9K0q|IIH|%=)Mh%n$mN-Q2ai5S2D8#Qk{?hXIhYzPD#UQCP2Gb^U$q{Xv{M3Eku97K3Q;AK<(F_ewVLYsJOIQvkW|ajmA98ynjs z>caQDrsZf~E&jN#fR-i^iiF(EvGX!6SgJ~XH_#x#CrNF))4m0h8JA@!$sM!(4EfLd zq7M1LOSrj=Kf_kqy~f+;h4y;5PD%@efp&BOY*$M7O3t#AsgZ@0u) zE-F~8T8d<0*>PX2qerIbQ&4&$| z@$*2YkNk1acC)ts3ns9E?>cDE|3jHXPXA!r$^(0byKeec9#Zky@=w$L`NhBf7itN9 Ag#Z8m literal 0 HcmV?d00001 diff --git a/eval/README.md b/eval/README.md index 0aba3d4..6d98282 100644 --- a/eval/README.md +++ b/eval/README.md @@ -56,7 +56,7 @@ Notes: accept the licence, or point `--tokenizer` at a local copy. - **GR00T** arches need `--stats-json /dataset_statistics.json` and an embodiment selected server-side via `VLA_GR00T_EMBODIMENT` (`new_embodiment` for N1.5, `libero_panda` for N1.6, - `libero_sim` for N1.7), plus `VLA_GR00T_BF16_WEIGHTS=1` to fit an 8 GB card. + `libero_sim` for N1.7). BF16 weights are the default, which is also what fits an 8 GB card. To sweep every model over `libero_object` tasks 0–9, use `eval/run_libero.sh -i `. @@ -66,7 +66,7 @@ So far only **GR00T-N1.6** is wired (the `gr00t-n1d6-bridge` checkpoint with the embodiment). Serve it, then drive from the SimplerEnv venv: ```bash -VLA_GR00T_BF16_WEIGHTS=1 VLA_GR00T_EMBODIMENT=oxe_widowx \ +VLA_GR00T_EMBODIMENT=oxe_widowx \ ./build/vla-server "$GR00T_N1D6_GGUF" eval/sim/simpler/simpler_uv/.venv/bin/python eval/client/run_simpler_client_direct.py \ diff --git a/eval/bitvla_ref/bench_bitvla_pytorch.py b/eval/bitvla_ref/bench_bitvla_pytorch.py new file mode 100644 index 0000000..920de0b --- /dev/null +++ b/eval/bitvla_ref/bench_bitvla_pytorch.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sim-free latency + memory benchmark for the upstream PyTorch BitVLA policy. + +Loads the model exactly the way ``run_libero_eval_bitnet.py`` does (same +``initialize_model``, same action head and proprio projector, same processor), +then replays one fixed LIBERO-object observation through ``get_action`` N +times. Dropping the simulator removes MuJoCo's EGL rendering from the GPU +memory accounting and its stepping from the wall clock, so the numbers here +are the policy's alone. + +The companion script ``run_libero_eval_bitnet_instrumented.py`` measures the +same two timers inside a real rollout; the two should agree on latency, and +differ on GPU memory by whatever the renderer holds. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from instrument import ( # noqa: E402 + MemoryProbe, + Timer, + parameter_report, + print_summary, + write_report, +) + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--pretrained-checkpoint", required=True) + p.add_argument("--task-suite-name", default="libero_object") + p.add_argument( + "--task-label", + default="pick up the black bowl between the plate and the ramekin and place it on the plate", + help="Instruction string; only its token count matters for latency.", + ) + p.add_argument("--n-steps", type=int, default=200, help="timed get_action calls") + p.add_argument( + "--warmup", + type=int, + default=0, + help="untimed calls before the window (0 = 10 eager / 20 compiled)", + ) + p.add_argument( + "--compile", + default="off", + choices=["off", "default", "reduce-overhead", "max-autotune"], + help="torch.compile inductor mode; 'off' runs eager", + ) + p.add_argument( + "--compile-target", + default="modules", + choices=["modules", "predict_action"], + help=( + "'modules' compiles the vision tower and the BitNet LM separately, " + "matching what the other reference policies in eval/pytorch_ref do; " + "'predict_action' compiles the whole policy entry point" + ), + ) + p.add_argument("--output", default="", help="path for the JSON report") + p.add_argument("--seed", type=int, default=7) + return p.parse_args() + + +def load_compile_helper(): + """Load eval/pytorch_ref/policies/torch_compile.py without its package. + + Importing it as ``policies.torch_compile`` would run the package __init__, + which claims HUGGINGFACE_HUB_CACHE and creates a weights dir. The module + itself only needs torch, so load it straight from the file and keep the two + stacks on one definition of "compiled". + """ + import importlib.util + + path = ( + Path(__file__).resolve().parents[1] + / "pytorch_ref" / "policies" / "torch_compile.py" + ) + spec = importlib.util.spec_from_file_location("vla_torch_compile", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def apply_compile(model, action_head, mode: str, target: str) -> list: + """Wrap the compile targets in place; returns the labels that were wrapped.""" + helper = load_compile_helper() + os.environ["VLA_TORCH_COMPILE"] = "1" + os.environ["VLA_TORCH_COMPILE_MODE"] = mode + + if target == "predict_action": + # predict_action ends in .cpu().numpy() and does .item() on token + # counts, so this graph-breaks hard; kept as a variant because it is + # the entry point a user would reach for first. + model.predict_action = helper.maybe_compile( + model.predict_action, tag="bitvla.predict_action" + ) + return ["predict_action"] + + # BitVLA never routes pixels through LlavaForConditionalGeneration.forward + # (predict_action calls get_image_features itself and passes pixel_values + # =None downstream), so the tower and the LM have to be compiled + # separately -- compiling the parent forward would miss the vision half. + model.vision_tower = helper.maybe_compile(model.vision_tower, tag="bitvla.vision_tower") + model.language_model = helper.maybe_compile(model.language_model, tag="bitvla.language_model") + return ["vision_tower", "language_model"] + + +def dynamo_stats() -> dict: + """How much of the model dynamo actually captured. + + maybe_compile() sets ``suppress_errors``, which turns an untraceable region + into a graph break instead of a hard failure. That is the right default for + measuring what a user gets, but it means "compiled" can quietly mean + "partly compiled" -- so count the breaks and report them. + """ + try: + from torch._dynamo.utils import counters + except ImportError: + return {} + breaks = dict(counters.get("graph_break", {})) + return { + "graph_breaks_total": sum(breaks.values()), + "graph_breaks_distinct": len(breaks), + "graph_break_reasons": dict(sorted(breaks.items(), key=lambda kv: -kv[1])[:10]), + "frames_ok": counters.get("frames", {}).get("ok", 0), + "frames_total": counters.get("frames", {}).get("total", 0), + } + + +def main() -> int: + args = parse_args() + + # Resolve caller-relative paths BEFORE chdir, or they land under + # third_party/BitVLA/openvla-oft instead of where the caller meant. + args.output = str(Path(args.output).resolve()) if args.output else "" + + # experiments.robot.* resolve through the editable openvla-oft install, but + # check_model_logic_mismatch() reads "./bitvla" relative to the CWD, so the + # loader only finds the reference modeling files from the repo root. + oft_root = Path(__file__).resolve().parents[2] / "third_party" / "BitVLA" / "openvla-oft" + os.chdir(oft_root) + # bitvla/model/bitvla_for_action_prediction.py imports `configuration_bit_vla` + # as a top-level module. Upstream's `pip install -e bitvla/` makes that work + # by rooting the distribution at bitvla/; the modern editable finder maps + # only the subpackages, so put the directory on the path explicitly. + sys.path.insert(0, str(oft_root / "bitvla")) + + from experiments.robot.libero.run_libero_eval_bitnet import ( + GenerateConfig, + initialize_model, + ) + from experiments.robot.robot_utils import get_action, get_image_resize_size, set_seed_everywhere + from bitvla.constants import ( + BITNET_ACTION_TOKEN_BEGIN_IDX, + BITNET_DEFAULT_IMAGE_TOKEN_IDX, + BITNET_IGNORE_INDEX, + BITNET_PROPRIO_PAD_IDX, + BITNET_STOP_INDEX, + ) + from prismatic.vla.constants import NUM_ACTIONS_CHUNK + + set_seed_everywhere(args.seed) + + cfg = GenerateConfig( + model_family="bitnet", + pretrained_checkpoint=args.pretrained_checkpoint, + task_suite_name=args.task_suite_name, + use_l1_regression=True, + use_diffusion=False, + use_film=False, + num_images_in_input=2, + use_proprio=True, + center_crop=True, + num_open_loop_steps=NUM_ACTIONS_CHUNK, + use_wandb=False, + seed=args.seed, + ) + + probe = MemoryProbe() + probe.start() + model, action_head, proprio_projector, noisy_action_projector, processor = initialize_model(cfg) + model.set_constant( + image_token_idx=BITNET_DEFAULT_IMAGE_TOKEN_IDX, + proprio_pad_idx=BITNET_PROPRIO_PAD_IDX, + ignore_idx=BITNET_IGNORE_INDEX, + action_token_begin_idx=BITNET_ACTION_TOKEN_BEGIN_IDX, + stop_index=BITNET_STOP_INDEX, + ) + probe.mark_weights_loaded() + + params = parameter_report( + { + "vla": model, + "action_head": action_head, + "proprio_projector": proprio_projector, + } + ) + + compiled_targets = [] + if args.compile != "off": + compiled_targets = apply_compile(model, action_head, args.compile, args.compile_target) + + # Time the forward on its own as well as the whole policy call: vla.cpp's + # server-reported latency excludes host-side image preprocessing, and here + # that preprocessing is a TensorFlow JPEG round-trip plus a lanczos3 + # resize, which is not a rounding error. + t_get_action = Timer("get_action") + t_forward = Timer("predict_action") + model.predict_action = t_forward.wrap(model.predict_action) + timed_get_action = t_get_action.wrap(get_action) + + resize_size = get_image_resize_size(cfg) + rng = np.random.default_rng(args.seed) + # env_img_res=256 in the LIBERO driver; resize_image_for_policy() takes it + # down to 224, so feed the benchmark the same 256x256 the sim would emit. + observation = { + "full_image": rng.integers(0, 256, size=(256, 256, 3), dtype=np.uint8), + "wrist_image": rng.integers(0, 256, size=(256, 256, 3), dtype=np.uint8), + "state": rng.uniform(-1.0, 1.0, size=(8,)).astype(np.float64), + } + print(f"[bench] resize_size={resize_size}, chunk={NUM_ACTIONS_CHUNK}, unnorm_key={cfg.unnorm_key}") + + def one_call(): + # get_action() mutates obs["state"] in place (normalize_proprio), so + # hand it a fresh copy each call or the proprio drifts across steps. + obs = {k: (v.copy() if isinstance(v, np.ndarray) else v) for k, v in observation.items()} + return timed_get_action( + cfg, + model, + obs, + args.task_label, + processor=processor, + action_head=action_head, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + use_film=cfg.use_film, + ) + + # Compilation happens on the first call and CUDA graphs need a few more to + # settle, so a compiled variant gets a longer warmup. Every warmup sample is + # discarded, which is what keeps the tens of seconds of inductor codegen out + # of the measured window. + warmup = args.warmup if args.warmup > 0 else (20 if compiled_targets else 10) + t0_warm = time.perf_counter() + for _ in range(warmup): + actions = one_call() + print( + f"[bench] warmup done ({warmup} calls in {time.perf_counter() - t0_warm:.1f}s); " + f"chunk length={len(actions)}" + ) + t_get_action.reset() + t_forward.reset() + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats() + + for i in range(args.n_steps): + one_call() + if (i + 1) % 50 == 0: + print(f"[bench] {i + 1}/{args.n_steps}") + + probe.finish() + + payload = { + "run": { + "mode": "sim_free", + "checkpoint": str(args.pretrained_checkpoint), + "task_suite": args.task_suite_name, + "n_steps": args.n_steps, + "warmup": warmup, + "chunk": NUM_ACTIONS_CHUNK, + "variant": f"compile-{args.compile}" if compiled_targets else "eager", + "compile_target": args.compile_target if compiled_targets else None, + "compiled_modules": compiled_targets, + "dynamo": dynamo_stats() if compiled_targets else {}, + "torch": torch.__version__, + "device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu", + "dtype": "bfloat16", + }, + "latency": { + "get_action": t_get_action.stats(), + "predict_action": t_forward.stats(), + }, + "memory": probe.report(), + "parameters": params, + } + print_summary(payload) + if args.output: + write_report(args.output, payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/bitvla_ref/instrument.py b/eval/bitvla_ref/instrument.py new file mode 100644 index 0000000..613ac05 --- /dev/null +++ b/eval/bitvla_ref/instrument.py @@ -0,0 +1,291 @@ +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Latency and memory instrumentation for the upstream PyTorch BitVLA stack. + +Two timers, because they answer different questions: + + * ``get_action`` — everything the control loop pays per query: the TF + JPEG round-trip + lanczos3 resize, the center crop, tokenization, and the + forward. This is BitVLA's analogue of ``select_action`` in the other + PyTorch reference servers under ``eval/pytorch_ref``. + * ``predict_action`` — the model forward alone. vla.cpp's server-reported + latency excludes host-side image preprocessing, so this is the number that + lines up with it. + +CUDA kernel launches are asynchronous, so both timers synchronize on each side +of the call; without that they would measure launch overhead, not compute. + +Memory is reported four ways because they are not interchangeable: + + * ``weights_bytes`` — allocator bytes resident right after the model, the + action head and the proprio projector are on the device. This is what the + checkpoint costs, and it is the number the BitVLA paper's "Memory Usage" + column is about — except the released checkpoint holds BF16 *master* + weights and quantizes online, so the measured value is the BF16 cost, not + the 1.58-bit one. + * ``peak_allocated`` — allocator high-water mark during the timed window + (weights + activations + KV cache). + * ``peak_reserved`` — what the caching allocator held from the driver. + * ``peak_vram`` — what the driver charges the process: reserved plus + the CUDA context, cuBLAS/cuDNN workspaces and any EGL rendering surfaces. + This is the number ``nvidia-smi`` shows, and the only one that reflects + what the GPU cannot give to anything else. Sampled on a background thread + and reported as a maximum, which is what ``mem_sampler_linux`` in + ``ci/lib/common.sh`` does for the vla.cpp server — so ``peak_vram_mib`` + here and in the ``*.mem.json`` files mean the same thing. +""" + +from __future__ import annotations + +import json +import os +import resource +import statistics +import subprocess +import threading +import time +from typing import Callable, Dict, List, Optional + +import torch + + +def _cuda_sync() -> None: + if torch.cuda.is_available(): + torch.cuda.synchronize() + + +class Timer: + """Accumulates per-call wall times (ms) around a wrapped callable.""" + + def __init__(self, name: str): + self.name = name + self.samples: List[float] = [] + + def wrap(self, fn: Callable) -> Callable: + def wrapper(*args, **kwargs): + _cuda_sync() + t0 = time.perf_counter() + out = fn(*args, **kwargs) + _cuda_sync() + self.samples.append(1000.0 * (time.perf_counter() - t0)) + return out + + return wrapper + + def reset(self) -> int: + """Drop samples collected so far; returns how many were discarded.""" + n = len(self.samples) + self.samples.clear() + return n + + def stats(self) -> Dict[str, float]: + s = sorted(self.samples) + if not s: + return {"n": 0} + + def pct(p: float) -> float: + if len(s) == 1: + return s[0] + idx = min(len(s) - 1, max(0, int(round(p * (len(s) - 1))))) + return s[idx] + + return { + "n": len(s), + "mean_ms": statistics.fmean(s), + "median_ms": statistics.median(s), + "min_ms": s[0], + "max_ms": s[-1], + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + "std_ms": statistics.pstdev(s) if len(s) > 1 else 0.0, + } + + +def nvidia_smi_process_mib(device_index: int = 0) -> Optional[float]: + """GPU memory the driver charges to this PID, in MiB. + + Falls back to ``None`` when nvidia-smi is unavailable or the PID has not + shown up in the compute-apps table yet. + """ + try: + out = subprocess.run( + [ + "nvidia-smi", + "--query-compute-apps=pid,used_memory", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=20, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + pid = str(os.getpid()) + for line in out.stdout.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) == 2 and parts[0] == pid: + try: + return float(parts[1]) + except ValueError: + return None + return None + + +def parameter_report(modules: Dict[str, torch.nn.Module]) -> Dict[str, object]: + """Per-module parameter counts and bytes, split by dtype. + + BitVLA ships BF16 master weights and ternarizes on the fly, so the byte + total here is the deployed cost of the released checkpoint — not the + 1.58-bit figure the paper quotes. + """ + per_module = {} + total_params = 0 + total_bytes = 0 + dtype_params: Dict[str, int] = {} + for name, mod in modules.items(): + if mod is None: + continue + n_params = 0 + n_bytes = 0 + for p in mod.parameters(): + n_params += p.numel() + n_bytes += p.numel() * p.element_size() + key = str(p.dtype).replace("torch.", "") + dtype_params[key] = dtype_params.get(key, 0) + p.numel() + for b in mod.buffers(): + n_bytes += b.numel() * b.element_size() + per_module[name] = {"params": n_params, "bytes": n_bytes} + total_params += n_params + total_bytes += n_bytes + return { + "per_module": per_module, + "total_params": total_params, + "total_bytes": total_bytes, + "params_by_dtype": dtype_params, + } + + +class MemoryProbe: + """Snapshots CUDA allocator state and samples driver-side VRAM.""" + + def __init__(self, device_index: int = 0, sample_interval_s: float = 0.2): + self.device_index = device_index + self.sample_interval_s = sample_interval_s + self.weights_bytes: Optional[int] = None + self.peak_allocated: Optional[int] = None + self.peak_reserved: Optional[int] = None + self.peak_vram_mib: Optional[float] = None + self.vram_samples = 0 + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + def _sample_loop(self) -> None: + # A single reading at the end would miss the high-water mark: the + # caching allocator does not return memory to the driver, but MuJoCo's + # renderer and cuBLAS workspaces come and go. + while not self._stop.is_set(): + mib = nvidia_smi_process_mib(self.device_index) + if mib is not None: + self.vram_samples += 1 + if self.peak_vram_mib is None or mib > self.peak_vram_mib: + self.peak_vram_mib = mib + self._stop.wait(self.sample_interval_s) + + def start(self) -> None: + if self._thread is not None: + return + self._thread = threading.Thread(target=self._sample_loop, daemon=True) + self._thread.start() + + def mark_weights_loaded(self) -> None: + """Record resident bytes with the model on device but nothing run yet.""" + if not torch.cuda.is_available(): + return + torch.cuda.synchronize() + self.weights_bytes = torch.cuda.memory_allocated(self.device_index) + torch.cuda.reset_peak_memory_stats(self.device_index) + + def finish(self) -> None: + if torch.cuda.is_available(): + torch.cuda.synchronize() + self.peak_allocated = torch.cuda.max_memory_allocated(self.device_index) + self.peak_reserved = torch.cuda.max_memory_reserved(self.device_index) + # One last reading before the sampler stops, so a short run that never + # completed a sampling tick still reports something. + mib = nvidia_smi_process_mib(self.device_index) + if mib is not None and (self.peak_vram_mib is None or mib > self.peak_vram_mib): + self.peak_vram_mib = mib + self.vram_samples += 1 + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=5.0) + + def report(self) -> Dict[str, object]: + mib = 1024.0 * 1024.0 + out: Dict[str, object] = {} + if self.weights_bytes is not None: + out["weights_mib"] = self.weights_bytes / mib + if self.peak_allocated is not None: + out["peak_allocated_mib"] = self.peak_allocated / mib + if self.peak_reserved is not None: + out["peak_reserved_mib"] = self.peak_reserved / mib + if self.peak_vram_mib is not None: + out["peak_vram_mib"] = self.peak_vram_mib + out["vram_samples"] = self.vram_samples + out["peak_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + return out + + +def write_report(path: str, payload: Dict[str, object]) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + with open(path, "w") as f: + json.dump(payload, f, indent=2, sort_keys=True) + print(f"[instrument] wrote {path}") + + +def print_summary(payload: Dict[str, object]) -> None: + print("\n" + "=" * 72) + print("BitVLA (PyTorch) — latency and memory") + print("=" * 72) + for key in ("get_action", "predict_action"): + st = payload.get("latency", {}).get(key) + if not st or not st.get("n"): + continue + print( + f"{key:<16} n={st['n']:<5d} mean {st['mean_ms']:7.2f} ms " + f"median {st['median_ms']:7.2f} p95 {st['p95_ms']:7.2f} " + f"min {st['min_ms']:7.2f} max {st['max_ms']:7.2f}" + ) + mem = payload.get("memory", {}) + for key, label in ( + ("weights_mib", "weights on device"), + ("peak_allocated_mib", "peak allocated"), + ("peak_reserved_mib", "peak reserved"), + ("peak_vram_mib", "peak VRAM (nvidia-smi)"), + ("peak_rss_mib", "peak host RSS"), + ): + if key in mem: + print(f"{label:<22} {mem[key]:9.1f} MiB") + params = payload.get("parameters", {}) + if params: + print( + f"{'parameters':<22} {params.get('total_params', 0) / 1e9:9.3f} B " + f"({params.get('total_bytes', 0) / (1024 ** 3):.2f} GiB on device)" + ) + for dt, n in sorted(params.get("params_by_dtype", {}).items()): + print(f" {dt:<20} {n / 1e9:9.3f} B params") + print("=" * 72 + "\n") diff --git a/eval/bitvla_ref/run_libero_bitvla_pytorch.py b/eval/bitvla_ref/run_libero_bitvla_pytorch.py new file mode 100644 index 0000000..f704a00 --- /dev/null +++ b/eval/bitvla_ref/run_libero_bitvla_pytorch.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run BitVLA's own LIBERO driver, instrumented for latency and memory. + +This does NOT reimplement the rollout: it imports upstream's +``experiments/robot/libero/run_libero_eval_bitnet.py`` and calls its +``eval_libero``, so the episode protocol, preprocessing and action +post-processing are upstream's byte for byte. What it adds is + + * a timer around ``get_action`` (the full policy query) and one around + ``model.predict_action`` (the forward alone), + * CUDA allocator snapshots taken right after the weights land on the device + and again at the end of the run, + * a JSON report next to the upstream text log. + +Note that the GPU numbers here include MuJoCo's offscreen renderer, which +shares the device with the policy. ``bench_bitvla_pytorch.py`` runs the same +model with no simulator attached, and the difference between the two is the +renderer's share. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from instrument import ( # noqa: E402 + MemoryProbe, + Timer, + parameter_report, + print_summary, + write_report, +) + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--pretrained-checkpoint", required=True) + p.add_argument("--task-suite-name", default="libero_object") + p.add_argument( + "--num-trials-per-task", + type=int, + default=1, + help="episodes per task; the suite has 10 tasks, so 1 gives a 10-episode sanity run", + ) + p.add_argument("--output", default="", help="path for the JSON report") + p.add_argument("--log-dir", default="", help="upstream text-log directory") + p.add_argument("--run-note", default="latency", help="upstream info_in_path tag") + p.add_argument("--save-videos", action="store_true", help="keep the MP4 rollouts") + p.add_argument( + "--warmup-calls", + type=int, + default=0, + help=( + "policy queries to discard before the timed window " + "(0 = 5 eager / 20 compiled; covers lazy CUDA init and inductor codegen)" + ), + ) + p.add_argument( + "--compile", + default="off", + choices=["off", "default", "reduce-overhead", "max-autotune"], + help="torch.compile inductor mode; 'off' runs eager", + ) + p.add_argument( + "--compile-target", + default="modules", + choices=["modules", "predict_action"], + help="see bench_bitvla_pytorch.py", + ) + p.add_argument("--seed", type=int, default=7) + return p.parse_args() + + +def main() -> int: + args = parse_args() + + # Resolve caller-relative paths BEFORE chdir, or they land under + # third_party/BitVLA/openvla-oft instead of where the caller meant. + args.output = str(Path(args.output).resolve()) if args.output else "" + args.log_dir = str(Path(args.log_dir).resolve()) if args.log_dir else "" + + # check_model_logic_mismatch() resolves "./bitvla" against the CWD. + repo_root = Path(__file__).resolve().parents[2] + oft_root = repo_root / "third_party" / "BitVLA" / "openvla-oft" + os.chdir(oft_root) + # See bench_bitvla_pytorch.py: `configuration_bit_vla` is imported as a + # top-level module by the BitVLA modeling file. + sys.path.insert(0, str(oft_root / "bitvla")) + + import experiments.robot.libero.run_libero_eval_bitnet as ev + from bench_bitvla_pytorch import apply_compile, dynamo_stats + + compiling = args.compile != "off" + warmup_calls = args.warmup_calls if args.warmup_calls > 0 else (20 if compiling else 5) + + probe = MemoryProbe() + probe.start() + t_get_action = Timer("get_action") + t_forward = Timer("predict_action") + state = { + "params": None, + "warmup_left": warmup_calls, + "reset_done": False, + "compiled": [], + } + + # eval_libero() builds the model itself, so hook the constructor to grab + # the post-load allocator snapshot and to wrap predict_action. + orig_initialize_model = ev.initialize_model + + def initialize_model(cfg): + model, action_head, proprio_projector, noisy_action_projector, processor = ( + orig_initialize_model(cfg) + ) + probe.mark_weights_loaded() + state["params"] = parameter_report( + {"vla": model, "action_head": action_head, "proprio_projector": proprio_projector} + ) + if compiling: + state["compiled"] = apply_compile( + model, action_head, args.compile, args.compile_target + ) + # Wrap after compiling: apply_compile may replace predict_action itself. + model.predict_action = t_forward.wrap(model.predict_action) + return model, action_head, proprio_projector, noisy_action_projector, processor + + ev.initialize_model = initialize_model + + # The first few queries pay lazy cuBLAS/cuDNN init and kernel autotuning; + # keeping them would put a multi-hundred-ms outlier in every percentile. + orig_get_action = ev.get_action + timed_get_action = t_get_action.wrap(orig_get_action) + + def get_action(*a, **kw): + out = timed_get_action(*a, **kw) + if state["warmup_left"] > 0: + state["warmup_left"] -= 1 + if state["warmup_left"] == 0: + dropped = (t_get_action.reset(), t_forward.reset()) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats() + state["reset_done"] = True + print(f"[instrument] warmup over, dropped {dropped[0]} samples") + return out + + ev.get_action = get_action + + if not args.save_videos: + ev.save_rollout_video = lambda *a, **kw: None + + log_dir = args.log_dir or str(repo_root / "outputs" / "bitvla_pytorch" / "logs") + + # eval_libero is draccus-wrapped, so it parses sys.argv itself. + sys.argv = [ + "run_libero_eval_bitnet.py", + "--pretrained_checkpoint", str(args.pretrained_checkpoint), + "--task_suite_name", args.task_suite_name, + "--model_family", "bitnet", + "--num_trials_per_task", str(args.num_trials_per_task), + "--use_wandb", "False", + "--local_log_dir", log_dir, + "--info_in_path", args.run_note, + "--seed", str(args.seed), + ] + + success_rate = ev.eval_libero() + probe.finish() + + payload = { + "run": { + "mode": "libero_rollout", + "checkpoint": str(args.pretrained_checkpoint), + "task_suite": args.task_suite_name, + "num_trials_per_task": args.num_trials_per_task, + "warmup_calls": warmup_calls, + "warmup_applied": state["reset_done"], + "torch": torch.__version__, + "device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu", + "dtype": "bfloat16", + "variant": f"compile-{args.compile}" if compiling else "eager", + "compile_target": args.compile_target if compiling else None, + "compiled_modules": state["compiled"], + "dynamo": dynamo_stats() if compiling else {}, + }, + "success_rate": success_rate, + "latency": { + "get_action": t_get_action.stats(), + "predict_action": t_forward.stats(), + }, + "memory": probe.report(), + "parameters": state["params"] or {}, + } + print(f"\n[instrument] LIBERO {args.task_suite_name} success rate: {success_rate * 100:.1f}%") + print_summary(payload) + if args.output: + write_report(args.output, payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/bitvla_ref/verify_compile_parity.py b/eval/bitvla_ref/verify_compile_parity.py new file mode 100644 index 0000000..d11b113 --- /dev/null +++ b/eval/bitvla_ref/verify_compile_parity.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check that torch.compile does not change BitVLA's actions. + +A 3x speedup on a model whose whole cost is online quantization is plausible — +inductor fuses the per-BitLinear absmean/absmax elementwise chains that eager +runs as dozens of separate kernels — but "fast" is only interesting if the +numbers still match. This loads the policy once, records the action chunk for +a fixed observation in eager, then compiles the same instance and records it +again, and reports the difference. + +Same process and same weights on both sides, so any delta is the compiled +kernels, not a reload. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from bench_bitvla_pytorch import apply_compile # noqa: E402 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--pretrained-checkpoint", required=True) + p.add_argument("--task-suite-name", default="libero_object") + p.add_argument( + "--task-label", + default="pick up the black bowl between the plate and the ramekin and place it on the plate", + ) + p.add_argument("--compile", default="default", choices=["default", "reduce-overhead", "max-autotune"]) + p.add_argument("--compile-target", default="modules", choices=["modules", "predict_action"]) + p.add_argument("--n-obs", type=int, default=4, help="distinct observations to compare") + p.add_argument( + "--real-obs", + action="store_true", + help=( + "draw observations from a real LIBERO episode instead of uniform " + "noise; noise is out of distribution and exaggerates the delta" + ), + ) + p.add_argument("--task-id", type=int, default=0, help="[--real-obs] LIBERO task") + p.add_argument("--seed", type=int, default=7) + return p.parse_args() + + +def real_observations(cfg, n: int, task_id: int, log=print): + """Step a LIBERO episode and keep every Nth observation. + + Uses upstream's own prepare_observation, so the images have been through + the same TF resize the policy sees at eval time. + """ + from libero.libero import benchmark + from experiments.robot.libero.libero_utils import get_libero_dummy_action, get_libero_env + from experiments.robot.libero.run_libero_eval_bitnet import prepare_observation + from experiments.robot.robot_utils import get_image_resize_size + + task_suite = benchmark.get_benchmark_dict()[cfg.task_suite_name]() + task = task_suite.get_task(task_id) + env, task_description = get_libero_env(task, cfg.model_family, resolution=cfg.env_img_res) + env.reset() + obs = env.set_init_state(task_suite.get_task_init_states(task_id)[0]) + + resize_size = get_image_resize_size(cfg) + out = [] + # num_steps_wait dummy actions first, same as the eval driver, so the + # objects have settled before anything is captured. + for t in range(cfg.num_steps_wait + n * 3): + if t < cfg.num_steps_wait: + obs, _, _, _ = env.step(get_libero_dummy_action(cfg.model_family)) + continue + if (t - cfg.num_steps_wait) % 3 == 0: + observation, _ = prepare_observation(obs, resize_size) + out.append(observation) + if len(out) == n: + break + obs, _, _, _ = env.step([0.0] * 6 + [-1.0]) + env.close() + log(f"[verify] collected {len(out)} real observations from '{task_description}'") + return out, task_description + + +def main() -> int: + args = parse_args() + + oft_root = Path(__file__).resolve().parents[2] / "third_party" / "BitVLA" / "openvla-oft" + os.chdir(oft_root) + sys.path.insert(0, str(oft_root / "bitvla")) + + from experiments.robot.libero.run_libero_eval_bitnet import GenerateConfig, initialize_model + from experiments.robot.robot_utils import get_action, set_seed_everywhere + from bitvla.constants import ( + BITNET_ACTION_TOKEN_BEGIN_IDX, + BITNET_DEFAULT_IMAGE_TOKEN_IDX, + BITNET_IGNORE_INDEX, + BITNET_PROPRIO_PAD_IDX, + BITNET_STOP_INDEX, + ) + from prismatic.vla.constants import NUM_ACTIONS_CHUNK + + set_seed_everywhere(args.seed) + cfg = GenerateConfig( + model_family="bitnet", + pretrained_checkpoint=args.pretrained_checkpoint, + task_suite_name=args.task_suite_name, + use_l1_regression=True, + num_images_in_input=2, + use_proprio=True, + center_crop=True, + num_open_loop_steps=NUM_ACTIONS_CHUNK, + use_wandb=False, + seed=args.seed, + ) + model, action_head, proprio_projector, noisy_action_projector, processor = initialize_model(cfg) + model.set_constant( + image_token_idx=BITNET_DEFAULT_IMAGE_TOKEN_IDX, + proprio_pad_idx=BITNET_PROPRIO_PAD_IDX, + ignore_idx=BITNET_IGNORE_INDEX, + action_token_begin_idx=BITNET_ACTION_TOKEN_BEGIN_IDX, + stop_index=BITNET_STOP_INDEX, + ) + + task_label = args.task_label + if args.real_obs: + observations, task_label = real_observations(cfg, args.n_obs, args.task_id) + else: + rng = np.random.default_rng(args.seed) + observations = [ + { + "full_image": rng.integers(0, 256, size=(256, 256, 3), dtype=np.uint8), + "wrist_image": rng.integers(0, 256, size=(256, 256, 3), dtype=np.uint8), + "state": rng.uniform(-1.0, 1.0, size=(8,)).astype(np.float64), + } + for _ in range(args.n_obs) + ] + + def run(obs): + # get_action normalizes obs["state"] in place. + o = {k: (v.copy() if isinstance(v, np.ndarray) else v) for k, v in obs.items()} + return np.stack( + get_action( + cfg, model, o, task_label, + processor=processor, + action_head=action_head, + proprio_projector=proprio_projector, + noisy_action_projector=noisy_action_projector, + use_film=cfg.use_film, + ) + ) + + eager = [run(o) for o in observations] + print(f"[verify] eager done ({len(eager)} observations, chunk shape {eager[0].shape})") + + apply_compile(model, action_head, args.compile, args.compile_target) + run(observations[0]) # trigger compilation, discard + compiled = [run(o) for o in observations] + print(f"[verify] compile-{args.compile} done") + + worst_abs = 0.0 + worst_rel = 0.0 + for i, (a, b) in enumerate(zip(eager, compiled)): + abs_d = float(np.max(np.abs(a - b))) + scale = float(np.max(np.abs(a))) or 1.0 + rel_d = abs_d / scale + worst_abs = max(worst_abs, abs_d) + worst_rel = max(worst_rel, rel_d) + print(f" obs {i}: max|Δ| = {abs_d:.3e} rel = {rel_d:.3e}") + + print() + print(f"[verify] observations: {'real LIBERO' if args.real_obs else 'uniform noise'}") + print(f"[verify] worst max|Δ| over {len(eager)} observations: {worst_abs:.3e}") + print(f"[verify] worst relative : {worst_rel:.3e}") + # There is no defensible tolerance here to pass/fail against. BitVLA + # ternarizes weights from an absmean scale computed inside the forward, so + # a 1-ulp change in that reduction moves a weight across a rounding + # boundary and flips it between -1/0/+1 -- a discrete change, not a drift. + # torch.compile reassociates those reductions, so some flips are expected + # and the per-action delta says little on its own. Success rate on the real + # suite is the arbiter; see run_libero_bitvla_pytorch.py --compile. + print("[verify] reported, not gated: see the report for why a tolerance would be arbitrary") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/collect_latency_compare.py b/eval/collect_latency_compare.py new file mode 100644 index 0000000..9413260 --- /dev/null +++ b/eval/collect_latency_compare.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build the vla.cpp vs torch.compile latency table from a sweep's JSONs. + +Reads outputs/latency_compare//.json (written by +eval/client/benchmark.py via eval/run_latency_compare.sh) and emits markdown. + +Every latency here is SERVER-SIDE inference time — the vla.cpp server reports +it per response, the PyTorch server times select_action in-process with CUDA +synchronization. Neither includes ZMQ transport or image serialization. + +Weight precision is read off the artifacts themselves rather than assumed: the +GGUF tensor-type histogram for vla.cpp, the safetensors dtype histogram for the +PyTorch checkpoint. The two stacks are NOT forced to matching precision — each +runs as shipped — so the dtype columns are load-bearing when reading the table. +""" + +from __future__ import annotations + +import argparse +import json +import struct +from collections import Counter +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +MODELS = ["smolvla", "pi0", "evo1", "gr00t_n1_5", "gr00t_n1_6", "gr00t_n1_7"] +VARIANTS = ["vla.cpp", "eager", "compile-default", "compile-reduce-overhead"] +TORCH_VARIANTS = ["eager", "compile-default", "compile-reduce-overhead"] + +# Where each stack's weights live, for the precision disclosure. +GGUF_ROOT = Path("/mnt/data/hf_data/vrfai") +CKPT_ROOT = Path("/mnt/data/hf_data") + +GGUF_PATHS = { + "smolvla": GGUF_ROOT / "smolvla-libero-gguf/smolvla-libero.gguf", + "pi0": GGUF_ROOT / "pi0-libero-finetuned-v044-gguf/pi0-libero-finetuned-v044.gguf", + "evo1": GGUF_ROOT / "evo1-libero-gguf/evo1-libero.gguf", + "gr00t_n1_5": GGUF_ROOT / "gr00tn1d5-libero-object-gguf/gr00tn1d5-libero-object.gguf", + "gr00t_n1_6": GGUF_ROOT / "gr00tn1d6-libero-gguf/gr00tn1d6-libero.gguf", + "gr00t_n1_7": GGUF_ROOT / "gr00tn1d7-libero-gguf/libero_object/gr00tn1d7-libero-object.gguf", +} + +CKPT_PATHS = { + "smolvla": CKPT_ROOT / "HuggingFaceVLA/smolvla_libero", + "pi0": CKPT_ROOT / "lerobot/pi0_libero_finetuned_v044", + "evo1": Path("/mnt/data/vla_sr_compare/weights/Evo1_LIBERO"), + "gr00t_n1_5": CKPT_ROOT / "liorbenhorin-nv/groot-libero_object-64_40000", + "gr00t_n1_6": CKPT_ROOT / "0xAnkitSingh/GR00T-N1.6-LIBERO", + "gr00t_n1_7": CKPT_ROOT / "nvidia/GR00T-N1.7-LIBERO/libero_object", +} + +# Runtime compute dtype, from the policy pipelines (not the on-disk weights): +# gr00t_n1_5/6/7 cast weights to bf16 before serving; evo1 runs its forward +# under torch.autocast(bf16); smolvla and pi0 run at the checkpoint's dtype. +TORCH_RUNTIME_DTYPE = { + "smolvla": "ckpt dtype", + "pi0": "ckpt dtype", + "evo1": "bf16 autocast", + "gr00t_n1_5": "bf16", + "gr00t_n1_6": "bf16", + "gr00t_n1_7": "bf16", +} + +# ggml_type enum -> name (only the values these GGUFs actually use). +GGML_TYPES = { + 0: "F32", 1: "F16", 2: "Q4_0", 3: "Q4_1", 6: "Q5_0", 7: "Q5_1", + 8: "Q8_0", 9: "Q8_1", 10: "Q2_K", 11: "Q3_K", 12: "Q4_K", 13: "Q5_K", + 14: "Q6_K", 15: "Q8_K", 16: "IQ2_XXS", 17: "IQ2_XS", 18: "IQ3_XXS", + 19: "IQ1_S", 20: "IQ4_NL", 21: "IQ3_S", 22: "IQ2_S", 23: "IQ4_XS", + 24: "I8", 25: "I16", 26: "I32", 27: "I64", 28: "F64", 29: "IQ1_M", + 30: "BF16", +} + + +def _read_gguf_tensor_types(path: Path) -> Counter | None: + """Parse a GGUF header and histogram its tensor types by element count. + + Hand-rolled rather than via the `gguf` package, which is not installed in + any of this repo's venvs. Only the header is read — no weights are loaded. + """ + try: + with path.open("rb") as f: + if f.read(4) != b"GGUF": + return None + version, = struct.unpack(" str: + n, = struct.unpack(" None: + if vtype == 8: # string + n, = struct.unpack(" Counter | None: + """Histogram safetensors dtypes by element count, from headers only.""" + files = sorted(ckpt_dir.glob("*.safetensors")) + if not files: + files = sorted(ckpt_dir.glob("**/*.safetensors")) + if not files: + return None + hist: Counter = Counter() + for fp in files: + try: + with fp.open("rb") as f: + n, = struct.unpack(" str: + """Render a dtype histogram as the dominant types by share of elements.""" + if not hist: + return "n/a" + total = sum(hist.values()) + if total == 0: + return "n/a" + parts = [] + for name, count in hist.most_common(top): + pct = 100.0 * count / total + if pct < 1.0: + continue + parts.append(f"{name} {pct:.0f}%") + return " + ".join(parts) if parts else "n/a" + + +def _failure_reason(root: Path, model: str, variant: str) -> str: + """Recover why a cell has no result, from that run's server log. + + Auto-extracted rather than hardcoded so the table stays honest if the + sweep is re-run and a previously failing configuration starts working. + """ + log = root / "_server_logs" / f"{model}.{variant}.log" + if not log.is_file(): + return "no server log — run did not start" + try: + text = log.read_text(errors="replace") + except OSError: + return "server log unreadable" + for line in text.splitlines(): + if "Error in server:" in line: + msg = line.split("Error in server:", 1)[1].strip() + if not msg: + continue + if "accessing tensor output of CUDAGraphs" in msg: + return ("CUDA-graph output overwritten by the next run — a tensor " + "escapes the compiled region and is reused across calls") + if "Dynamo failed to run FX node" in msg and "flash_attn" in msg: + return "dynamo cannot trace flash-attn's varlen op" + return msg[:180] + if "StopIteration" in text: + return "StopIteration" + return "failed — see server log" + + +def _load(root: Path, model: str, variant: str) -> dict | None: + p = root / model / f"{variant}.json" + if not p.is_file(): + return None + try: + return json.loads(p.read_text()) + except ValueError: + return None + + +def _server_mean(stats: dict | None) -> float | None: + if not stats: + return None + s = stats.get("server_ms") + return s.get("mean") if s else None + + +def _server_med(stats: dict | None) -> float | None: + if not stats: + return None + s = stats.get("server_ms") + return s.get("median") if s else None + + +def _cell(v: float | None) -> str: + return f"{v:.1f}" if v is not None else "—" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", type=Path, + default=REPO_ROOT / "outputs" / "latency_compare") + ap.add_argument("--output", type=Path, default=None, + help="Write markdown here (default: stdout).") + args = ap.parse_args() + + data = {m: {v: _load(args.root, m, v) for v in VARIANTS} for m in MODELS} + + lines: list[str] = [] + add = lines.append + + add("# vla.cpp vs torch.compile — inference latency") + add("") + add("Server-side inference time per prediction, mean over the timed window, " + "on one LIBERO-object episode stream.") + add("") + + # --- main table -------------------------------------------------------- + add("Each cell is **mean (median)**. Several models have a heavy right tail, " + "so the two differ materially — see the distribution table below.") + add("") + add("| Model | vla.cpp | PyTorch eager | compile (default) | compile (reduce-overhead) | best PyTorch | vla.cpp vs best PyTorch |") + add("|---|---:|---:|---:|---:|---|---:|") + for m in MODELS: + cpp = _server_mean(data[m]["vla.cpp"]) + torch_vals = {v: _server_mean(data[m][v]) for v in TORCH_VARIANTS} + present = {k: v for k, v in torch_vals.items() if v is not None} + best_name, best_val = (min(present.items(), key=lambda kv: kv[1]) + if present else (None, None)) + if cpp is not None and best_val: + ratio = best_val / cpp + verdict = (f"**{ratio:.2f}× faster**" if ratio > 1.0 + else f"{1.0 / ratio:.2f}× slower") + else: + verdict = "—" + def both(variant: str) -> str: + mean = _server_mean(data[m][variant]) + med = _server_med(data[m][variant]) + if mean is None: + return "—" + return f"{mean:.1f} ({med:.1f})" if med is not None else f"{mean:.1f}" + + add(f"| {m} | {both('vla.cpp')} | {both('eager')} | " + f"{both('compile-default')} | " + f"{both('compile-reduce-overhead')} | " + f"{best_name or '—'} | {verdict} |") + add("") + add("All figures in milliseconds. Lower is better. " + "The final column is best-PyTorch ÷ vla.cpp: >1× means vla.cpp wins.") + add("") + + # Scoreboard, computed rather than asserted. + wins, losses, ties = [], [], [] + for m in MODELS: + cpp = _server_mean(data[m]["vla.cpp"]) + present = [v for v in (_server_mean(data[m][x]) for x in TORCH_VARIANTS) + if v is not None] + if cpp is None or not present: + continue + ratio = min(present) / cpp + (wins if ratio > 1.02 else losses if ratio < 0.98 else ties).append(m) + add(f"**Scoreboard** — against each model's best available PyTorch variant, " + f"vla.cpp is faster on {len(wins)}/{len(wins) + len(losses) + len(ties)} " + f"models ({', '.join(wins) or 'none'}), slower on {len(losses)} " + f"({', '.join(losses) or 'none'})" + + (f", tied on {len(ties)} ({', '.join(ties)})" if ties else "") + ".") + add("") + add("Note the spread: vla.cpp's mean and median sit within ~1% of each " + "other on every model, while the PyTorch variants carry long tails on " + "several. For a control loop the tail is often what matters.") + add("") + + # --- what torch.compile bought ---------------------------------------- + add("## What torch.compile bought") + add("") + add("| Model | eager | best compiled | compile speedup |") + add("|---|---:|---:|---:|") + for m in MODELS: + eager = _server_mean(data[m]["eager"]) + comp = {v: _server_mean(data[m][v]) + for v in ("compile-default", "compile-reduce-overhead")} + comp = {k: v for k, v in comp.items() if v is not None} + if not eager or not comp: + add(f"| {m} | {_cell(eager)} | — | — |") + continue + bn, bv = min(comp.items(), key=lambda kv: kv[1]) + add(f"| {m} | {eager:.1f} | {bv:.1f} ({bn.replace('compile-', '')}) | " + f"{eager / bv:.2f}× |") + add("") + + # --- precision disclosure --------------------------------------------- + add("## Weight precision (as shipped — not forced to match)") + add("") + add("| Model | vla.cpp GGUF weights | PyTorch checkpoint weights | PyTorch compute |") + add("|---|---|---|---|") + for m in MODELS: + gg = _fmt_hist(_read_gguf_tensor_types(GGUF_PATHS[m])) if GGUF_PATHS[m].is_file() else "n/a" + st = _fmt_hist(_read_safetensors_dtypes(CKPT_PATHS[m])) if CKPT_PATHS[m].exists() else "n/a" + if st == "n/a": + st = CKPT_DTYPE_FALLBACK.get(m, "n/a") + add(f"| {m} | {gg} | {st} | {TORCH_RUNTIME_DTYPE[m]} |") + add("") + add("Percentages are share of tensor **elements**, not bytes. Each stack " + "runs in its intended configuration, so a row where the two dtypes " + "differ is comparing deployments, not kernels.") + add("") + + # --- validity ---------------------------------------------------------- + # Verified 2026-08-05 by reading the GGUF metadata KVs against each + # PyTorch checkpoint config (falling back to its base model where the + # finetune omits a key). Latency on a flow-matching policy scales with + # denoise steps and chunk length, so a mismatch here would mean the table + # measured configuration rather than implementation. + add("## Work per call (verified equal on both stacks)") + add("") + add("| Model | denoise steps | chunk (timesteps) | action dim |") + add("|---|---:|---:|---:|") + for m, steps, chunk, dim in [ + ("smolvla", 10, 50, 32), + ("pi0", 10, 50, 32), + ("evo1", 32, 50, 24), + ("gr00t_n1_5", 4, 16, 32), + ("gr00t_n1_6", 4, 50, 128), + ("gr00t_n1_7", 4, 40, 132), + ]: + add(f"| {m} | {steps} | {chunk} | {dim} |") + add("") + add("Both stacks agree on every row, so each pair of numbers compares the " + "same computation. Two caveats on reading the PyTorch configs: Evo-1's " + "step count is forced to 32 in the pipeline (its checkpoint omits the " + "key, and the code default is 50), and GR00T-N1.5 inherits 4 steps / " + "16-timestep horizon from its base model — lerobot's `chunk_size=50` " + "is a wrapper value, not what the DiT head generates.") + add("") + add("Every call is also a genuine forward pass, not an action-queue pop: " + "per-cell `min` sits within 3% of `median` on all variants except " + "GR00T-N1.5's compiled ones (recompilation, not replay). A pop costs " + "only preprocessing, so any replay would drag `min` far below " + "`median` — that is exactly how the GR00T-N1.5 bug was caught.") + add("") + + # --- spread ------------------------------------------------------------ + add("## Distribution (median / p95, ms)") + add("") + add("| Model | vla.cpp | eager | compile (default) | compile (reduce-overhead) |") + add("|---|---|---|---|---|") + for m in MODELS: + cells = [] + for v in VARIANTS: + s = (data[m][v] or {}).get("server_ms") + cells.append(f"{s['median']:.1f} / {s['p95']:.1f}" if s else "—") + add(f"| {m} | " + " | ".join(cells) + " |") + add("") + + # --- provenance -------------------------------------------------------- + add("## Run configuration") + add("") + any_stats = next((data[m][v] for m in MODELS for v in VARIANTS + if data[m][v] is not None), None) + if any_stats: + add(f"- Timed calls per cell: **{any_stats.get('n_steps')}** " + f"(warmup excluded from the reported samples)") + add(f"- Task: `{any_stats.get('task')}` / task_{any_stats.get('task_id')}") + add(f"- `n_action_steps = {any_stats.get('n_action_steps')}` on both " + f"stacks, so every call is a real forward pass rather than an " + f"action-queue pop") + add("- Metric: server-side inference only — excludes ZMQ transport and " + "image serialization") + add("- PyTorch timing is CUDA-synchronized on both sides of `select_action`") + add("") + + # --- missing cells ----------------------------------------------------- + missing = [(m, v) for m in MODELS for v in VARIANTS if data[m][v] is None] + if missing: + add("## Configurations that did not run") + add("") + add("| Model | Variant | Why |") + add("|---|---|---|") + for m, v in missing: + add(f"| {m} | {v} | {_failure_reason(args.root, m, v)} |") + add("") + add("These are reported rather than silently dropped: a variant that " + "cannot run is a real property of that model on this stack, not a " + "gap in the measurement.") + add("") + + out = "\n".join(lines) + if args.output: + args.output.write_text(out) + print(f"wrote {args.output}") + else: + print(out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/collect_libero_per_model.py b/eval/collect_libero_per_model.py new file mode 100755 index 0000000..056c37e --- /dev/null +++ b/eval/collect_libero_per_model.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Aggregate the BitVLA multi-suite LIBERO sweep produced by run_libero_bitvla.sh. + +Unlike collect_libero_results.py (many models, ONE suite), this sweep is ONE +model (bitvla) across SEVERAL suites — each suite under its own subdir: + + /bitvla_

{r['suite']}") + L.append("") + L.append("| Task | Successes | Terminated | SR | client/step (ms) |") + L.append("|---|---:|---:|---:|---:|") + for tid in sorted(r["per_task"]): + t = r["per_task"][tid] + sr = t["successes"] / t["n_episodes"] if t["n_episodes"] else 0.0 + L.append(f"| task_{tid} | {t['successes']}/{t['n_episodes']} | " + f"{t['skipped']}/{t['n_episodes']} | {sr:.2%} | {t['inf_ms']:.2f} |") + L.append("") + L.append("
") + L.append("") + return "\n".join(L) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--sweep", type=Path, default=DEFAULT_SWEEP, + help=f"sweep root (default: {DEFAULT_SWEEP})") + ap.add_argument("--md", type=Path, default=None, metavar="PATH", + help="markdown report path (default: /report.md)") + ap.add_argument("--no-md", action="store_true", help="do not write a markdown report") + args = ap.parse_args() + + if not args.sweep.is_dir(): + print(f"ERROR: sweep dir not found: {args.sweep}", file=sys.stderr) + return 1 + + server_logs_dir = args.sweep / "_server_logs" + suite_dirs = sorted(p for p in args.sweep.iterdir() + if p.is_dir() and not p.name.startswith(("_", "."))) + + rows: list[dict] = [] + for d in suite_dirs: + c = collect_suite(d) + if c is None: + print(f"warning: no summaries under {d}", file=sys.stderr) + continue + agg = aggregate(c["per_task"]) + rows.append({ + "label": d.name, + "suite": c["suite"], + "per_task": c["per_task"], + "agg": agg, + "server": parse_server_log(server_logs_dir / f"{d.name}.log"), + "mem": parse_mem_json(server_logs_dir / f"{d.name}.mem.json"), + }) + + if not rows: + print("No results found.", file=sys.stderr) + return 1 + rows.sort(key=lambda r: r["suite"]) + + print(f"Sweep: {args.sweep}") + print(f"Model: bitvla ({len(rows)} suite(s))") + print() + hdr = (f"{'suite':<16} {'n_act':>6} {'tasks':>6} {'success':>10} {'terminated':>12} " + f"{'SR':>8} {'client/step':>12} {'client/call':>12}") + print(hdr) + print("-" * len(hdr)) + for r in rows: + a = r["agg"] + n_act = a["n_action_steps"] + n_act_str = str(n_act) if n_act is not None else "?" + per_call = f"{a['avg_inf_ms'] * n_act:>12.2f}" if n_act is not None else f"{'?':>12}" + print(f"{r['suite']:<16} {n_act_str:>6} {a['n_tasks']:>6} " + f"{a['total_succ']:>4}/{a['total_eps']:<5} " + f"{a['total_skip']:>5}/{a['total_eps']:<6} " + f"{a['sr']:>7.2%} {a['avg_inf_ms']:>12.2f} {per_call}") + tot_succ = sum(r["agg"]["total_succ"] for r in rows) + tot_eps = sum(r["agg"]["total_eps"] for r in rows) + if tot_eps: + print("-" * len(hdr)) + print(f"{'ALL':<16} {'':>6} {sum(r['agg']['n_tasks'] for r in rows):>6} " + f"{tot_succ:>4}/{tot_eps:<5} {'':>12} {tot_succ / tot_eps:>7.2%}") + + if not args.no_md: + md_path = args.md if args.md is not None else (args.sweep / "report.md") + md_path.write_text(render_markdown(args.sweep, rows), encoding="utf-8") + print() + print(f"Markdown report written to: {md_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/collect_solver_displacement.py b/eval/collect_solver_displacement.py new file mode 100755 index 0000000..94c64dd --- /dev/null +++ b/eval/collect_solver_displacement.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Analyse the fixed-noise action chunks written by run_solver_step_displacement.sh. + +For each T, reports the action chunk's displacement against the T=4 reference +chunk (the checkpoint default): max absolute deviation, relative RMS, and +cosine similarity, plus the measured per-call latency. +""" + +from __future__ import annotations + +import argparse +import math +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OUT = REPO_ROOT / "outputs" / "solver_sweep" / "displacement" + +BENCH_RE = re.compile(r"predict\(\) over (\d+) iters: min=([\d.]+) ms\s+avg=([\d.]+) ms") +SPLIT_RE = re.compile(r"last split: vision=([\d.]+)\s+inference=([\d.]+)\s+total=([\d.]+) ms") +REF_T = 4 + + +def read_actions(path: Path) -> list[float]: + # model_load banners land on stdout ahead of the payload, so locate the + # header rather than assuming it is the first line. + lines = path.read_text().splitlines() + idx = next((i for i, l in enumerate(lines) if l.startswith("action_len=")), None) + if idx is None: + raise ValueError(f"{path}: no action_len header") + n = int(lines[idx].split("=", 1)[1]) + if n == 0: + raise ValueError(f"{path}: empty action chunk (action_len=0) -- " + "usually an image-size mismatch; check the .timing.txt") + vals = [float(x) for x in lines[idx + 1 : idx + 1 + n]] + if len(vals) != n: + raise ValueError(f"{path}: expected {n} values, got {len(vals)}") + return vals + + +def read_timing(path: Path) -> dict: + text = path.read_text() if path.is_file() else "" + out = {} + if (m := BENCH_RE.search(text)): + out["min_ms"], out["avg_ms"] = float(m.group(2)), float(m.group(3)) + if (m := SPLIT_RE.search(text)): + out["vision_ms"], out["inf_ms"] = float(m.group(1)), float(m.group(2)) + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("-o", "--out", type=Path, default=DEFAULT_OUT) + args = ap.parse_args() + out: Path = args.out + if not out.is_dir(): + print(f"ERROR: no such directory: {out}", file=sys.stderr) + return 1 + + files = sorted(out.glob("T*.actions.txt"), + key=lambda p: int(p.name.split(".")[0][1:])) + if not files: + print(f"ERROR: no T*.actions.txt under {out}", file=sys.stderr) + return 1 + + chunks = {int(f.name.split(".")[0][1:]): read_actions(f) for f in files} + if REF_T not in chunks: + print(f"ERROR: reference T={REF_T} missing", file=sys.stderr) + return 1 + ref = chunks[REF_T] + ref_rms = math.sqrt(sum(v * v for v in ref) / len(ref)) + + hdr = (f"{'T':>3} {'max|d| vs T=4':>13} {'rel RMS':>9} {'cosine':>10} " + f"{'avg ms':>7} {'vision':>7} {'inf':>7}") + print(hdr) + print("-" * len(hdr)) + rows = [] + for t, ch in chunks.items(): + if len(ch) != len(ref): + print(f"ERROR: T={t} chunk length {len(ch)} != ref {len(ref)}", file=sys.stderr) + return 1 + diffs = [a - b for a, b in zip(ch, ref)] + max_abs = max(abs(d) for d in diffs) + rms = math.sqrt(sum(d * d for d in diffs) / len(diffs)) + rel_rms = rms / ref_rms if ref_rms else float("nan") + dot = sum(a * b for a, b in zip(ch, ref)) + na = math.sqrt(sum(a * a for a in ch)) + nb = math.sqrt(sum(b * b for b in ref)) + cos = dot / (na * nb) if na and nb else float("nan") + tm = read_timing(out / f"T{t}.timing.txt") + rows.append((t, max_abs, rel_rms, cos, tm)) + print(f"{t:>3} {max_abs:>13.4e} {rel_rms:>9.3e} {cos:>10.8f} " + f"{tm.get('avg_ms', float('nan')):>7.2f} " + f"{tm.get('vision_ms', float('nan')):>7.2f} " + f"{tm.get('inf_ms', float('nan')):>7.2f}") + + md = [ + "# Solver-step displacement — GR00T-N1.7, fixed noise", + "", + "Action chunk at each T against the T=4 checkpoint default, from", + "`vla_predict_check` (fixed images / language / state / noise).", + f"Chunk is {len(ref)} values; `rel RMS` is RMS(delta) / RMS(reference chunk).", + "", + "| T | max abs dev vs T=4 | rel RMS | cosine | avg ms | vision ms | inference ms |", + "|---:|---:|---:|---:|---:|---:|---:|", + ] + for t, ma, rr, cos, tm in rows: + md.append( + f"| {t} | {ma:.4e} | {rr:.3e} | {cos:.8f} | " + f"{tm.get('avg_ms', float('nan')):.2f} | " + f"{tm.get('vision_ms', float('nan')):.2f} | " + f"{tm.get('inf_ms', float('nan')):.2f} |") + report = out / "displacement.md" + report.write_text("\n".join(md) + "\n") + print(f"\nWrote {report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/collect_solver_sweep.py b/eval/collect_solver_sweep.py new file mode 100644 index 0000000..706c8bc --- /dev/null +++ b/eval/collect_solver_sweep.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Aggregate the solver-step (T) sweep written by run_solver_step_sweep.sh. + +Reads the per-task summaries of both stacks at each T: + + /T/vla_cpp/gr00t_n1_7/gr00t_n1_7/libero_object/task_/summary.txt + /T/pytorch/gr00t_n1_7/gr00t_n1_7/libero_object/task_/summary.txt + +and reports, per T, each stack's success rate with a Wilson 95% interval, the +delta in percentage points with a two-proportion z-test, and the per-step +latency. Episodes the env terminated mid-step count as failures, matching +collect_sr_compare.py. + +Prints a table and writes a markdown report. +""" + +from __future__ import annotations + +import argparse +import math +import re +import sys +from datetime import datetime +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_ROOT = REPO_ROOT / "outputs" / "solver_sweep" + +SUCCESS_RE = re.compile(r"Success rate:\s*[\d.]+%\s*\((\d+)/(\d+)\)") +SKIPPED_RE = re.compile(r"Skipped \(terminated mid-step\):\s*(\d+)/(\d+)") +INF_RE = re.compile(r"Average inference time per step:\s*([\d.]+)\s*ms") +NACT_RE = re.compile(r"n_action_steps:\s*(\d+)") + +ARCH = "gr00t_n1_7" +SUITE = "libero_object" + + +def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """Wilson score interval for a binomial proportion, in percent.""" + if n == 0: + return (0.0, 0.0) + p = k / n + denom = 1.0 + z * z / n + centre = (p + z * z / (2 * n)) / denom + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom + return (100.0 * max(0.0, centre - half), 100.0 * min(1.0, centre + half)) + + +def two_proportion_p(k1: int, n1: int, k2: int, n2: int) -> float: + """Two-sided p-value for H0: p1 == p2, pooled-variance z-test.""" + if n1 == 0 or n2 == 0: + return float("nan") + p_pool = (k1 + k2) / (n1 + n2) + se = math.sqrt(p_pool * (1 - p_pool) * (1 / n1 + 1 / n2)) + if se == 0.0: + return 1.0 + z = (k1 / n1 - k2 / n2) / se + # Two-sided normal tail via erfc. + return math.erfc(abs(z) / math.sqrt(2.0)) + + +def read_cell(stack_dir: Path) -> dict | None: + """Sum the 10 per-task summaries under one / directory.""" + suite_dir = stack_dir / ARCH / ARCH / SUITE + if not suite_dir.is_dir(): + return None + succ = total = skipped = 0 + inf_ms: list[float] = [] + n_act: set[int] = set() + tasks = 0 + for task_dir in sorted(suite_dir.glob("task_*")): + summary = task_dir / "summary.txt" + if not summary.is_file(): + continue + text = summary.read_text() + m = SUCCESS_RE.search(text) + if not m: + continue + tasks += 1 + succ += int(m.group(1)) + total += int(m.group(2)) + if (s := SKIPPED_RE.search(text)): + skipped += int(s.group(1)) + if (i := INF_RE.search(text)): + inf_ms.append(float(i.group(1))) + if (a := NACT_RE.search(text)): + n_act.add(int(a.group(1))) + if total == 0: + return None + return { + "succ": succ, + "total": total, + "skipped": skipped, + "tasks": tasks, + "step_ms": sum(inf_ms) / len(inf_ms) if inf_ms else float("nan"), + "n_act": sorted(n_act), + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("-o", "--root", type=Path, default=DEFAULT_ROOT, + help=f"sweep results root (default: {DEFAULT_ROOT})") + ap.add_argument("-r", "--report", type=Path, default=None, + help="markdown report path (default: /solver_sweep.md)") + args = ap.parse_args() + + root: Path = args.root + if not root.is_dir(): + print(f"ERROR: no such directory: {root}", file=sys.stderr) + return 1 + + t_dirs = sorted(root.glob("T*"), key=lambda p: int(p.name[1:])) + if not t_dirs: + print(f"ERROR: no T directories under {root}", file=sys.stderr) + return 1 + + rows = [] + for t_dir in t_dirs: + t = int(t_dir.name[1:]) + cpp = read_cell(t_dir / "vla_cpp") + pt = read_cell(t_dir / "pytorch") + rows.append((t, cpp, pt)) + + hdr = (f"{'T':>3} {'vla.cpp SR':>22} {'PyTorch SR':>22} " + f"{'delta':>8} {'p':>6} {'cpp ms/step':>11} {'pt ms/step':>10}") + print(hdr) + print("-" * len(hdr)) + + lines_md = [ + "# Solver-step sweep — GR00T-N1.7, libero_object", + "", + f"- Generated: {datetime.now():%Y-%m-%d}", + f"- Root: `{root}`", + "- `T` is the flow-matching solver-step count (checkpoint default 4),", + " set on both stacks by `VLA_NUM_STEPS`.", + "- `n_action_steps` = 16 on both stacks at every T, so chunk-replay", + " cadence is held fixed and T is the only variable.", + "- Intervals are Wilson 95%; `p` is a two-proportion test, vla.cpp vs PyTorch.", + "", + "| T | vla.cpp SR (Wilson 95%) | PyTorch SR (Wilson 95%) | delta (pp) | p | vla.cpp ms/step | PyTorch ms/step |", + "|---:|---|---|---:|---:|---:|---:|", + ] + + for t, cpp, pt in rows: + def fmt(c): + if c is None: + return "—", "—" + lo, hi = wilson(c["succ"], c["total"]) + return (f"{100.0 * c['succ'] / c['total']:.1f}% " + f"({c['succ']}/{c['total']}) [{lo:.1f}, {hi:.1f}]", + f"{c['step_ms']:.2f}") + cpp_sr, cpp_ms = fmt(cpp) + pt_sr, pt_ms = fmt(pt) + if cpp and pt: + delta = 100.0 * (cpp["succ"] / cpp["total"] - pt["succ"] / pt["total"]) + p = two_proportion_p(cpp["succ"], cpp["total"], pt["succ"], pt["total"]) + delta_s, p_s = f"{delta:+.1f}", f"{p:.2f}" + else: + delta_s, p_s = "—", "—" + print(f"{t:>3} {cpp_sr:>22} {pt_sr:>22} {delta_s:>8} {p_s:>6} " + f"{cpp_ms:>11} {pt_ms:>10}") + lines_md.append( + f"| {t} | {cpp_sr} | {pt_sr} | {delta_s} | {p_s} | {cpp_ms} | {pt_ms} |") + + # Terminated-episode audit: a nonzero count anywhere invalidates the cell. + lines_md += ["", "## Terminated-episode audit", "", + "| T | vla.cpp skipped | PyTorch skipped | vla.cpp n_act | PyTorch n_act |", + "|---:|---:|---:|---|---|"] + for t, cpp, pt in rows: + cs = str(cpp["skipped"]) if cpp else "—" + ps = str(pt["skipped"]) if pt else "—" + cn = ",".join(map(str, cpp["n_act"])) if cpp else "—" + pn = ",".join(map(str, pt["n_act"])) if pt else "—" + lines_md.append(f"| {t} | {cs} | {ps} | {cn} | {pn} |") + + report = args.report or (root / "solver_sweep.md") + report.write_text("\n".join(lines_md) + "\n") + print(f"\nWrote {report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/collect_sr_compare.py b/eval/collect_sr_compare.py new file mode 100644 index 0000000..0e7d688 --- /dev/null +++ b/eval/collect_sr_compare.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Aggregate the side-by-side LIBERO sweep written by run_libero_compare.sh. + +Reads both phases' per-task `summary.txt` files: + + /pytorch////task_/summary.txt + /vla_cpp////task_/summary.txt + +and reports, per model, the PyTorch reference success rate next to the vla.cpp +GGUF success rate plus the delta in percentage points. Episodes the env +terminated mid-step ("skipped") count as failures, matching +collect_libero_results.py. + +Prints a table and writes a markdown report. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from datetime import datetime +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_ROOT = REPO_ROOT / "outputs" / "sr_compare" + +TASK_RE = re.compile(r"Task:\s*(\S+?)/task_(\d+)") +SUCCESS_RE = re.compile(r"Success rate:\s*[\d.]+%\s*\((\d+)/(\d+)\)") +SKIPPED_RE = re.compile(r"Skipped \(terminated mid-step\):\s*(\d+)/(\d+)") +INF_RE = re.compile(r"Average inference time per step:\s*([\d.]+)\s*ms") +NACT_RE = re.compile(r"n_action_steps:\s*(\d+)") + +# Models with no PyTorch wrapper in eval/pytorch_ref get their reference number +# from the upstream paper instead of a local run. +PAPER_SR = { + "bitvla": (99.6, "BitVLA paper, LIBERO-Object, 50 ep x 10 tasks"), +} + +# Display order; anything else found is appended alphabetically. +MODEL_ORDER = [ + "smolvla", "pi0", "evo1", "bitvla", + "gr00t_n1_5", "gr00t_n1_6", "gr00t_n1_7", +] + + +def parse_summary(path: Path) -> dict | None: + text = path.read_text() + m_s = SUCCESS_RE.search(text) + m_k = SKIPPED_RE.search(text) + if not (m_s and m_k): + print(f"WARN: unparsable summary, skipping: {path}", file=sys.stderr) + return None + successes, counted = int(m_s.group(1)), int(m_s.group(2)) + skipped, n_episodes = int(m_k.group(1)), int(m_k.group(2)) + m_t = TASK_RE.search(text) + m_i = INF_RE.search(text) + m_n = NACT_RE.search(text) + return { + "successes": successes, + "counted": counted, + "skipped": skipped, + "n_episodes": n_episodes, + "suite": m_t.group(1) if m_t else None, + "task_id": int(m_t.group(2)) if m_t else int(path.parent.name.split("_")[-1]), + "inference_ms": float(m_i.group(1)) if m_i else None, + "n_action_steps": int(m_n.group(1)) if m_n else None, + } + + +def collect_phase(phase_dir: Path) -> dict[str, dict]: + """model -> {tasks: {id: rec}, successes, episodes, inference_ms, n_action_steps}""" + out: dict[str, dict] = {} + if not phase_dir.is_dir(): + return out + for summary in sorted(phase_dir.glob("*/*/*/task_*/summary.txt")): + model = summary.parents[3].name + rec = parse_summary(summary) + if rec is None: + continue + entry = out.setdefault( + model, {"tasks": {}, "successes": 0, "episodes": 0, + "inf": [], "n_action_steps": set(), "suites": set()} + ) + if rec["task_id"] in entry["tasks"]: + print(f"WARN: duplicate task_{rec['task_id']} for {model}: {summary}", + file=sys.stderr) + entry["tasks"][rec["task_id"]] = rec + entry["successes"] += rec["successes"] + entry["episodes"] += rec["n_episodes"] + if rec["inference_ms"] is not None: + entry["inf"].append(rec["inference_ms"]) + if rec["n_action_steps"] is not None: + entry["n_action_steps"].add(rec["n_action_steps"]) + if rec["suite"]: + entry["suites"].add(rec["suite"]) + return out + + +def sr(entry: dict) -> float | None: + return 100.0 * entry["successes"] / entry["episodes"] if entry["episodes"] else None + + +def fmt_sr(entry: dict | None) -> str: + if entry is None or not entry["episodes"]: + return "—" + return f"{sr(entry):.1f}% ({entry['successes']}/{entry['episodes']})" + + +def fmt_nact(entry: dict | None) -> str: + if entry is None or not entry["n_action_steps"]: + return "—" + return "/".join(str(n) for n in sorted(entry["n_action_steps"])) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("-o", "--root", type=Path, default=DEFAULT_ROOT, + help=f"sweep root written by run_libero_compare.sh (default: {DEFAULT_ROOT})") + ap.add_argument("--out", type=Path, default=None, + help="markdown report path (default: /sr_compare.md)") + args = ap.parse_args() + + root: Path = args.root + if not root.is_dir(): + print(f"ERROR: sweep root not found: {root}", file=sys.stderr) + return 1 + + pt = collect_phase(root / "pytorch") + cpp = collect_phase(root / "vla_cpp") + if not pt and not cpp: + print(f"ERROR: no summary.txt found under {root}", file=sys.stderr) + return 1 + + models = [m for m in MODEL_ORDER if m in pt or m in cpp] + models += sorted(set(pt) | set(cpp) - set(models) - set(MODEL_ORDER)) + seen, ordered = set(), [] + for m in models: + if m not in seen: + seen.add(m) + ordered.append(m) + + suites = {s for e in list(pt.values()) + list(cpp.values()) for s in e["suites"]} + suite = ", ".join(sorted(suites)) if suites else "libero_object" + + lines: list[str] = [] + lines.append("# LIBERO success rate — PyTorch reference vs vla.cpp GGUF") + lines.append("") + lines.append(f"- Generated: {datetime.now().isoformat(timespec='seconds')}") + lines.append(f"- Suite: `{suite}`, tasks 0–9") + lines.append(f"- Sweep root: `{root}`") + lines.append("") + lines.append("Both stacks ran the same LIBERO env (`eval/sim/libero/libero_env.py`), " + "seed 42, 500-step cap, and the same per-arch action-chunk replay " + "(`n_act` column). Episodes the env terminated mid-step count as failures.") + lines.append("") + lines.append("| Model | n_act | PyTorch SR | vla.cpp SR | Δ (pp) | PyTorch ms/step | vla.cpp ms/step |") + lines.append("|---|---:|---:|---:|---:|---:|---:|") + + for m in ordered: + p, c = pt.get(m), cpp.get(m) + if p is None and m in PAPER_SR: + paper, _ = PAPER_SR[m] + pt_cell = f"{paper:.1f}%¹" + delta = f"{sr(c) - paper:+.1f}" if c and c["episodes"] else "—" + else: + pt_cell = fmt_sr(p) + delta = (f"{sr(c) - sr(p):+.1f}" + if p and c and p["episodes"] and c["episodes"] else "—") + # Prefer the vla.cpp value, but fall back to PyTorch's while only one + # phase has run — fmt_nact returns "—", which is truthy, so `or` here + # would never reach the fallback. + nact = fmt_nact(c) if (c and c["n_action_steps"]) else fmt_nact(p) + p_ms = f"{sum(p['inf']) / len(p['inf']):.1f}" if p and p["inf"] else "—" + c_ms = f"{sum(c['inf']) / len(c['inf']):.1f}" if c and c["inf"] else "—" + lines.append(f"| `{m}` | {nact} | {pt_cell} | {fmt_sr(c)} | {delta} | {p_ms} | {c_ms} |") + + if any(m in PAPER_SR for m in ordered if m not in pt): + lines.append("") + for m in ordered: + if m not in pt and m in PAPER_SR: + lines.append(f"¹ `{m}`: no PyTorch wrapper in `eval/pytorch_ref`; " + f"reference is the published number ({PAPER_SR[m][1]}), " + f"not a local run — the episode protocol differs.") + + # Per-task breakdown + lines.append("") + lines.append("## Per-task success (successes / episodes)") + lines.append("") + header = "| Model | Stack | " + " | ".join(f"t{i}" for i in range(10)) + " | total |" + lines.append(header) + lines.append("|---|---|" + "---:|" * 11) + for m in ordered: + for label, entry in (("PyTorch", pt.get(m)), ("vla.cpp", cpp.get(m))): + if entry is None: + continue + cells = [] + for i in range(10): + r = entry["tasks"].get(i) + cells.append(f"{r['successes']}/{r['n_episodes']}" if r else "—") + cells.append(f"**{entry['successes']}/{entry['episodes']}**") + lines.append(f"| `{m}` | {label} | " + " | ".join(cells) + " |") + + missing = [m for m in ordered if m in pt and m in cpp + and pt[m]["episodes"] != cpp[m]["episodes"]] + if missing: + lines.append("") + lines.append("> **Uneven episode counts** (one stack ran fewer episodes — the " + "delta above is not like-for-like): " + + ", ".join(f"`{m}`" for m in missing)) + + report = "\n".join(lines) + "\n" + out_path = args.out or (root / "sr_compare.md") + out_path.write_text(report) + print(report) + print(f"[wrote] {out_path}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/compare_act_dtype.py b/eval/compare_act_dtype.py new file mode 100644 index 0000000..4a7219e --- /dev/null +++ b/eval/compare_act_dtype.py @@ -0,0 +1,175 @@ +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Action-for-action comparison of two vla-server configurations. + +Used to check what a numerics change (BF16 activations, flash attention, ...) +actually does to the predicted actions, independently of whether it changes a +LIBERO episode's outcome. + +The two servers must see byte-identical inputs, which a live environment cannot +guarantee: the second server's actions steer the sim somewhere else and every +later observation diverges for reasons that have nothing to do with the kernel. +So `record` drives the env once and dumps the observation stream, and `replay` +feeds that fixed stream to any server. VLA_FIXED_NOISE_SEED pins the +flow-matching noise on top (see VlaCppClient._maybe_add_fixed_noise), leaving +the arithmetic as the only thing that differs. + + record: python compare_act_dtype.py record --addr ... --out ref.npz + replay: python compare_act_dtype.py replay --addr ... --obs ref.npz --out b.npz + diff: python compare_act_dtype.py diff a.npz b.npz +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "client")) + + +def _client(addr: str, arch: str, tokenizer: str | None): + from client.vla_cpp_client import VlaCppClient + kw = {"arch": arch, "n_action_steps": 1} + if tokenizer: + kw["tokenizer_name"] = tokenizer + return VlaCppClient(addr, **kw) + + +def _adapter(arch: str, client): + from client.adapters import Evo1PipelineAdapter + if arch != "evo1": + raise SystemExit(f"only evo1 is wired up here, got {arch!r}") + return Evo1PipelineAdapter(client=client) + + +def _obs_key(obs: dict) -> dict: + """Keep only what the evo1 request path reads, as plain arrays.""" + return { + "image": np.stack([np.asarray(im, dtype=np.uint8) for im in obs["image"]]), + "image_mask": np.asarray(obs.get("image_mask", [1] * len(obs["image"])), dtype=np.int32), + "state": np.asarray(obs["state"], dtype=np.float32), + "prompt": str(obs.get("prompt", "")), + } + + +def cmd_record(args) -> int: + import gymnasium as gym + import sim.libero # noqa: F401 registers the envs + + client = _client(args.addr, args.arch, args.tokenizer) + adapter = _adapter(args.arch, client) + env = gym.make(f"{args.task}/task_{args.task_id}", video_fps=30, + output_video_dir=str(args.out.parent / "_cmp_videos"), + video_view_mode="single-view") + obs, _ = env.reset(seed=args.seed) + + frames, actions = [], [] + for i in range(args.n_steps): + # record the model-space observation, which is what replay must reproduce + parsed = adapter.parse_observation(obs) + frames.append(_obs_key(parsed)) + chunk = client.get_action(parsed) + actions.append(np.asarray(chunk, dtype=np.float32)) + try: + obs, _r, done, trunc, _ = env.step(adapter.parse_action(chunk)) + except ValueError: + obs, _ = env.reset(seed=args.seed) + continue + if done or trunc: + obs, _ = env.reset(seed=args.seed) + + np.savez_compressed( + args.out, + images=np.stack([f["image"] for f in frames]), + image_masks=np.stack([f["image_mask"] for f in frames]), + states=np.stack([f["state"] for f in frames]), + prompts=np.array([f["prompt"] for f in frames]), + actions=np.stack(actions), + ) + print(f"recorded {len(frames)} steps -> {args.out}") + return 0 + + +def cmd_replay(args) -> int: + d = np.load(args.obs, allow_pickle=True) + client = _client(args.addr, args.arch, args.tokenizer) + + actions = [] + for i in range(len(d["images"])): + obs = { + "image": list(d["images"][i]), + "image_mask": list(d["image_masks"][i]), + "state": d["states"][i], + "prompt": str(d["prompts"][i]), + } + actions.append(np.asarray(client.get_action(obs), dtype=np.float32)) + + np.savez_compressed(args.out, actions=np.stack(actions)) + print(f"replayed {len(actions)} steps -> {args.out}") + return 0 + + +def cmd_diff(args) -> int: + a = np.load(args.a)["actions"].astype(np.float64) + b = np.load(args.b)["actions"].astype(np.float64) + if a.shape != b.shape: + print(f"shape mismatch: {a.shape} vs {b.shape}") + return 1 + d = np.abs(a - b) + scale = np.abs(a).mean() + print(f"steps : {a.shape[0]} action dim {a.shape[1:]}") + print(f"mean |a| : {scale:.6f}") + print(f"max |diff| : {d.max():.6e}") + print(f"mean |diff| : {d.mean():.6e}") + print(f"p99 |diff| : {np.percentile(d, 99):.6e}") + print(f"rel mean : {d.mean() / scale:.3e}") + print(f"exact equal : {bool(np.array_equal(a, b))}") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + for name in ("record", "replay"): + p = sub.add_parser(name) + p.add_argument("--addr", required=True) + p.add_argument("--arch", default="evo1") + p.add_argument("--tokenizer", default=None) + p.add_argument("--out", type=Path, required=True) + if name == "record": + p.add_argument("--task", default="libero_object") + p.add_argument("--task-id", type=int, default=0) + p.add_argument("--n-steps", type=int, default=20) + p.add_argument("--seed", type=int, default=42) + else: + p.add_argument("--obs", type=Path, required=True) + + p = sub.add_parser("diff") + p.add_argument("a", type=Path) + p.add_argument("b", type=Path) + + args = ap.parse_args() + return {"record": cmd_record, "replay": cmd_replay, "diff": cmd_diff}[args.cmd](args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/plot_solver_sweep.py b/eval/plot_solver_sweep.py new file mode 100644 index 0000000..af929d1 --- /dev/null +++ b/eval/plot_solver_sweep.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Render the E7 solver-step sweep figure from the collected sweep results. + +Two panels sharing the T axis -- success on the left, latency on the right. +Deliberately NOT a dual-axis chart: two measures on two scales sharing one plot +would invent a correlation that is not in the data. + +Reads outputs/solver_sweep/ via collect_solver_sweep.py's own parser, so the +figure and the table cannot drift apart. + + python eval/plot_solver_sweep.py [-o ] [-f ] +""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# House style, matching paper/replan_sweep.py and roofline.py so all three +# figures read as one set: DejaVu Serif (ships with matplotlib, so it renders +# identically everywhere), inward ticks, four spines, dotted grid. +# +# Colours are the paper's Okabe-Ito pair, reusing the same two hues in the same +# roles as replan_sweep: blue = the primary/success series, orange = the second. +# Re-validated for this figure against its real surface (white paper): +# validate_palette.js "#0072B2,#E69F00" --mode light --surface "#ffffff" +# -> CVD dE 29.2, normal-vision dE 36.2 (both clear), but orange is 2.25:1 +# against white, a contrast WARN. The skill's relief rule applies and is +# discharged two ways: marker shape carries identity alongside hue +# (circle vs square), and the caption prints the per-T values, so nothing +# depends on seeing the orange against the page. +VLA_CPP = "#0072B2" # blue (replan_sweep SUCCESS_COLOR) +PYTORCH = "#E69F00" # orange (replan_sweep RATE_COLOR) +SURFACE = "#ffffff" +GRID = "#bbbbbb" +AXIS = "#333333" + +N_ACT = 16 # chunk replay; client/step * N_ACT = per call + + +def set_research_style() -> None: + """Verbatim from paper/replan_sweep.py, so the figures share one look.""" + plt.rcParams.update({ + "font.family": "serif", + "font.serif": ["DejaVu Serif", "Times New Roman", "Nimbus Roman No9 L"], + "mathtext.fontset": "dejavuserif", + "font.size": 10.5, + "axes.titlesize": 11, + "axes.labelsize": 10.5, + "xtick.labelsize": 9, + "ytick.labelsize": 9, + "legend.fontsize": 9, + "axes.linewidth": 0.9, + "axes.edgecolor": AXIS, + "axes.axisbelow": True, + "xtick.direction": "in", + "ytick.direction": "in", + "xtick.major.size": 4, + "ytick.major.size": 4, + "xtick.minor.size": 2.2, + "ytick.minor.size": 2.2, + "figure.facecolor": "white", + "savefig.facecolor": "white", + "savefig.dpi": 220, + # Keep text as text (TrueType) in PDF/PS so it stays editable. + "pdf.fonttype": 42, + "ps.fonttype": 42, + }) + + +def load_collector(): + spec = importlib.util.spec_from_file_location( + "collect_solver_sweep", REPO_ROOT / "eval" / "collect_solver_sweep.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def style_axes(ax): + ax.set_facecolor(SURFACE) + # House style keeps all four spines (replan_sweep / roofline). + ax.grid(True, which="major", ls=":", lw=0.5, color=GRID, alpha=0.7) + ax.set_axisbelow(True) + + +def series(ax, x, y, color, marker, ls, label=None, lo=None, hi=None, + dodge=1.0, line=True): + """One series. `line=False` gives a dot plot. + + Panel A is trendless repeated measurement, so it gets no connecting line: + joining those points would draw a zigzag the data does not contain. Marker + shape doubles the hue, so identity survives the orange's low contrast. + """ + xd = [v * dodge for v in x] + if lo is not None: + err = [[y[i] - lo[i] for i in range(len(y))], + [hi[i] - y[i] for i in range(len(y))]] + ax.errorbar(xd, y, yerr=err, fmt="none", ecolor=color, + elinewidth=1.3, capsize=3, capthick=1.3, zorder=5) + if line: + ax.plot(xd, y, ls=ls, color=color, lw=1.8, zorder=4) + ax.plot(xd, y, marker, color=color, ms=5, markeredgecolor="white", + markeredgewidth=0.8, ls="none", zorder=6, label=label) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("-o", "--root", type=Path, + default=REPO_ROOT / "outputs" / "solver_sweep") + ap.add_argument("-f", "--out", type=Path, + default=REPO_ROOT / "docs" / "corl-paper" / "solver_sweep.pdf") + args = ap.parse_args() + + C = load_collector() + t_dirs = sorted(args.root.glob("T*"), key=lambda p: int(p.name[1:])) + t_dirs = [p for p in t_dirs if p.is_dir() and p.name[1:].isdigit()] + if not t_dirs: + print(f"ERROR: no T dirs under {args.root}", file=sys.stderr) + return 1 + + T, cpp_sr, cpp_lo, cpp_hi, pt_sr, pt_lo, pt_hi, cpp_ms, pt_ms = ([] for _ in range(9)) + for d in t_dirs: + cpp, pt = C.read_cell(d / "vla_cpp"), C.read_cell(d / "pytorch") + if not cpp or not pt: + print(f"WARNING: skipping incomplete cell {d.name}", file=sys.stderr) + continue + T.append(int(d.name[1:])) + for cell, sr, lo, hi, ms in ((cpp, cpp_sr, cpp_lo, cpp_hi, cpp_ms), + (pt, pt_sr, pt_lo, pt_hi, pt_ms)): + p = 100.0 * cell["succ"] / cell["total"] + l, h = C.wilson(cell["succ"], cell["total"]) + sr.append(p); lo.append(l); hi.append(h) + ms.append(cell["step_ms"] * N_ACT) + if not T: + print("ERROR: no complete cells", file=sys.stderr) + return 1 + + set_research_style() + fig, (axA, axB) = plt.subplots(1, 2, figsize=(7.0, 3.1)) + + # -- Panel A: success --------------------------------------------------- + # Small x dodge so the two Wilson intervals never sit on top of each other. + series(axA, T, cpp_sr, VLA_CPP, "o", "-", "vla.cpp", + cpp_lo, cpp_hi, dodge=0.965, line=False) + series(axA, T, pt_sr, PYTORCH, "s", "--", "PyTorch reference", + pt_lo, pt_hi, dodge=1.035, line=False) + # Plain "%" -- matplotlib is not LaTeX here, so "\%" would render literally. + axA.set_ylabel("LIBERO-Object success (%)") + axA.set_ylim(84, 101) + axA.set_yticks([85, 90, 95, 100]) + + # -- Panel B: latency --------------------------------------------------- + series(axB, T, cpp_ms, VLA_CPP, "o", "-") + series(axB, T, pt_ms, PYTORCH, "s", "--") + axB.set_ylabel("Latency per prediction (ms)") + axB.set_ylim(0, 215) + # Selective direct labels: the endpoints only, where the gap is widest. + # These are also the contrast relief for the orange series. + for val, color in ((pt_ms[-1], PYTORCH), (cpp_ms[-1], VLA_CPP)): + axB.annotate(f"{val:.0f}", (T[-1], val), textcoords="offset points", + xytext=(7, -3), ha="left", fontsize=9, color=color) + + for ax, title in ((axA, "Success is flat in $T$; every interval overlaps"), + (axB, "Latency grows with $T$; vla.cpp leads throughout")): + style_axes(ax) + ax.set_xscale("log", base=2) + ax.set_xticks(T) + ax.set_xticklabels([str(t) for t in T]) + ax.minorticks_off() + ax.set_xlim(T[0] * 0.85, T[-1] * (1.45 if ax is axB else 1.12)) + ax.set_xlabel("Solver steps $T$") + ax.set_title(title, fontsize=10, pad=6) + + # One frameless legend under both panels, as in replan_sweep. + h, l = axA.get_legend_handles_labels() + fig.legend(h, l, loc="lower center", ncol=2, frameon=False, + bbox_to_anchor=(0.5, -0.02), handletextpad=0.5) + + fig.tight_layout(rect=(0, 0.07, 1, 1.0), w_pad=2.0) + args.out.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(args.out, bbox_inches="tight") + png = args.out.with_suffix(".png") + fig.savefig(png, dpi=220, bbox_inches="tight") + print(f"Wrote {args.out}\nWrote {png}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/pytorch_ref/client/__init__.py b/eval/pytorch_ref/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eval/pytorch_ref/client/run_libero_eval.py b/eval/pytorch_ref/client/run_libero_eval.py new file mode 100644 index 0000000..1fe3142 --- /dev/null +++ b/eval/pytorch_ref/client/run_libero_eval.py @@ -0,0 +1,152 @@ +import sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import sim.libero # noqa: F401 to ensure the environments are registered with Gym +from utils.service import RobotInferenceClient +from utils.sim_adapters.libero import LIBEROSimAdapter + +import time +import argparse +import gymnasium as gym + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--task", type=str, default="libero_object/task_0", + help="The task to test on, select one of ['libero_10', 'libero_spatial', 'libero_object', 'libero_goal', 'libero_90'] " + "with the corresponding task_id, e.g. 'libero_object/task_0'." + ) + parser.add_argument( + "--n-episodes", type=int, default=30, + help="The number of episodes to run for evaluation" + ) + parser.add_argument( + "--fps", type=int, default=30, + help="The frames per second (FPS) for the output video recording of each episode" + ) + parser.add_argument( + "--output-dir", type=str, default="outputs", + help="The directory to save the output videos. Each episode will be saved as a separate video file in this directory." + ) + parser.add_argument( + "--view-mode", + choices=["single-view", "multi-view"], + default="multi-view", + help="single-view: write one camera key, multi-view: side-by-side front+wrist views", + ) + parser.add_argument( + "--host", type=str, default="localhost", + help="Host of the inference server (run_server.py)." + ) + parser.add_argument( + "--port", type=int, default=5555, + help="Port of the inference server (run_server.py)." + ) + parser.add_argument( + "--seed", type=int, default=42, + help="Seed for the LIBERO environment reset/init-state rollout (default: 42)." + ) + parser.add_argument( + "--out-name", type=str, default=None, + help="Name of the per-model output subdir (default: the arch reported by " + "the server). GR00T N1.6 and N1.7 both report arch 'gr00t', so pass " + "this to keep their results apart." + ) + parser.add_argument( + "--n-action-steps", type=int, default=None, + help="Recorded in summary.txt for provenance only — the replay itself is " + "server-side (see server/*.py --n-action-steps). Pass the same value " + "you gave the server so the comparison report can verify the two " + "stacks ran the same control cadence." + ) + args = parser.parse_args() + + client = RobotInferenceClient(host=args.host, port=args.port, api_token=None) + client = LIBEROSimAdapter(client=client) + + out_name = args.out_name or client.arch + output_dir = Path(args.output_dir) / out_name / args.task + output_dir.mkdir(parents=True, exist_ok=True) + + # control_mode = "absolute" if client.arch == "gr00t" else "relative" # GR00T-N1.6 is trained with absolute control, while the others are trained with relative control + + env = gym.make( + args.task, + seed=args.seed, + video_fps=args.fps, + output_video_dir=output_dir, + video_view_mode=args.view_mode, + # control_mode=control_mode, + ) + + # Run Simulations + success_count, inference_times = 0.0, [] + skipped = 0 + for episode in range(args.n_episodes): + print(f"*** Episode {episode + 1}/{args.n_episodes}") + + client.reset() + obs, info = env.reset() + run_times, step_id = [], 0 + episode_aborted = False + done = False + truncated = False + reward = 0.0 + + while True: + # Get action from the policy + t0 = time.time() + action = client.get_action(obs) + run_times.append(time.time() - t0) + + try: + obs, reward, done, truncated, info = env.step(action) + except ValueError as e: + # robosuite raises this when the underlying env's `done` flag was + # set on the previous step but the lerobot/LIBERO wrapper didn't + # propagate it as terminated=True (so our auto-reset path didn't + # fire). Skip this episode and move on. + if "terminated episode" not in str(e): + raise + print(f"- Episode aborted (env reported terminated mid-step): {e}") + episode_aborted = True + break + #print(f"- Step {step_id}: reward={reward:.2f}, done={done}, truncated={truncated}, info={info}") + step_id += 1 + + if done or truncated or episode_aborted: + avg_t = sum(run_times) / len(run_times) + inference_times.append(avg_t) + success_count += info.get("is_success", 0.0) + + print(f"- Episode finished after {step_id} steps.") + print(f"- Final reward: {reward:.2f}") + print(f"- Episode Information:\n{info}") + print(f"- Average inference time per step: {round(1000 * avg_t, 2)} ms") + break + + if episode_aborted: + skipped += 1 + + env.close() + counted = max(1, args.n_episodes - skipped) + avg_inf_ms = (round(1000 * sum(inference_times) / len(inference_times), 2) + if inference_times else 0.0) + # Same field layout as vla.cpp's eval/client/run_sim_client_direct.py, so + # eval/collect_libero_results.py parses both stacks' outputs unchanged. + with open(output_dir / "summary.txt", "w") as f: + f.write(f"Arch: {out_name}\n") + f.write(f"Task: {args.task}\n") + f.write(f"n_action_steps: {args.n_action_steps}\n") + f.write("Task Description: " + env.task_description + "\n") + f.write(f"Success rate: {success_count / counted:.2%} ({int(success_count)}/{counted})\n") + f.write(f"Skipped (terminated mid-step): {skipped}/{args.n_episodes}\n") + f.write(f"Average inference time per step: {avg_inf_ms} ms\n") + + print("*** All episodes completed.") + print(f"- Success rate: {success_count / counted:.2%} ({int(success_count)}/{counted})") + print(f"- Skipped (terminated mid-step): {skipped}/{args.n_episodes}") + print(f"- Saved videos to: {output_dir.resolve()}") diff --git a/eval/pytorch_ref/policies/__init__.py b/eval/pytorch_ref/policies/__init__.py new file mode 100644 index 0000000..3628e6d --- /dev/null +++ b/eval/pytorch_ref/policies/__init__.py @@ -0,0 +1,13 @@ +import os +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# Where local checkpoint dirs live. Checkpoints for the SR comparison are large +# and sit on the data volume, so allow an env override instead of hard-coding a +# `weights/` dir inside the repo. +POLICY_DIR = Path(os.environ.get("VLA_POLICY_DIR", ROOT / "weights")) +POLICY_DIR.mkdir(parents=True, exist_ok=True) + +# Only claim the hub cache when the caller has not already placed it somewhere. +os.environ.setdefault("HUGGINGFACE_HUB_CACHE", str(POLICY_DIR)) diff --git a/eval/pytorch_ref/policies/action_chunk.py b/eval/pytorch_ref/policies/action_chunk.py new file mode 100644 index 0000000..fdd7253 --- /dev/null +++ b/eval/pytorch_ref/policies/action_chunk.py @@ -0,0 +1,50 @@ +"""Action-chunk replay queue, shared by the GR00T pipelines. + +The GR00T N1.6 / N1.7 pipelines predict a whole chunk `[B, T, D]` per forward +but the LIBERO client re-queries every env step, so without a queue they +re-predict from scratch at every step (`n_action_steps == 1`) and throw away +`T - 1` of every chunk. + +vla.cpp's client replays `n_action_steps` actions from each chunk before +re-querying `vla-server`. To make the PyTorch reference and the GGUF port +comparable on success rate, this queue reproduces that cadence on the PyTorch +side: predict a chunk, hand out the first `n_action_steps` timesteps one per +call, then predict again from the observation current at that moment. + +Timesteps are kept in the pipeline's native `dict[str, ndarray[B, T, D]]` shape +(sliced to `T == 1`) so the client-side parsers stay unchanged. +""" + +from __future__ import annotations + +from collections import deque +from typing import Any + +import numpy as np + + +class ActionChunkQueue: + """Hands out one timestep per call from a predicted `[B, T, D]` chunk.""" + + def __init__(self, n_action_steps: int = 1) -> None: + if n_action_steps < 1: + raise ValueError(f"n_action_steps must be >= 1, got {n_action_steps}") + self.n_action_steps = n_action_steps + self._queue: deque[dict[str, np.ndarray]] = deque() + + def clear(self) -> None: + self._queue.clear() + + @property + def empty(self) -> bool: + return not self._queue + + def fill(self, chunk: dict[str, Any]) -> None: + """Split a `dict[str, ndarray[B, T, D]]` chunk into per-timestep dicts.""" + horizon = min(arr.shape[1] for arr in chunk.values()) + n = min(self.n_action_steps, horizon) + for t in range(n): + self._queue.append({k: v[:, t : t + 1] for k, v in chunk.items()}) + + def pop(self) -> dict[str, np.ndarray]: + return self._queue.popleft() diff --git a/eval/pytorch_ref/policies/evo1/__init__.py b/eval/pytorch_ref/policies/evo1/__init__.py new file mode 100644 index 0000000..9ba5253 --- /dev/null +++ b/eval/pytorch_ref/policies/evo1/__init__.py @@ -0,0 +1,139 @@ +from typing import Any +import os +import json +import cv2 +import torch +import numpy as np +from PIL import Image +from contextlib import nullcontext +from torchvision.transforms import ToTensor +from huggingface_hub import snapshot_download + +from policies import POLICY_DIR +from policies.evo1.policy import EVO1Policy +from policies.evo1.normalizer import EVO1Normalizer +from policies.torch_compile import maybe_compile + + +class EVO1PolicyPipeline: + arch_type: str = "evo1" + + def __init__( + self, + model_id: str = "MINT-SJTU/Evo1_LIBERO", + device: str | torch.device | None = None, + n_action_steps: int | None = None, + ) -> None: + model_name = model_id.split("/")[-1] + ckpt_dir = POLICY_DIR / model_name + if not ckpt_dir.exists(): + ckpt_dir = snapshot_download( + repo_id=model_id, + local_dir=str(ckpt_dir), + local_dir_use_symlinks=False + ) + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + # Chunk replay. EVO1Policy keeps its own deque and re-predicts once + # n_action_steps of the horizon-50 chunk have been consumed — the same + # cadence as vla.cpp's client-side --n-action-steps. + self._policy, self._normalizer = self.load_policy_and_normalizer( + ckpt_dir, self.device, n_action_steps + ) + if n_action_steps is not None: + print(f"[evo1] n_action_steps={self._policy.n_action_steps}", flush=True) + self.to_tensor = ToTensor() + + # Optional torch.compile. Evo-1 splits cleanly into two hot spots: the + # InternVL3 embedder (one forward per prediction) and the flow-matching + # action head (num_inference_timesteps=50 forwards per prediction). The + # head is the dominant cost and the better compile target, but both are + # wrapped so the compiled variant is not measuring a half-compiled model. + self._policy.action_head = maybe_compile( + self._policy.action_head, tag="evo1.action_head", + ) + # The embedder is driven through submodule calls — extract_feature() + # invokes vision_model(...) and the wrapper then calls language_model(...) + # directly — so InternVL3's own forward() never runs. Compiling the top + # level module would be a silent no-op; these two are the calls that + # actually execute. + self._policy.embedder.model.vision_model = maybe_compile( + self._policy.embedder.model.vision_model, tag="evo1.embedder.vision_model", + ) + self._policy.embedder.model.language_model = maybe_compile( + self._policy.embedder.model.language_model, tag="evo1.embedder.language_model", + ) + + def get_arch(self) -> dict[str, str]: + return {"arch": self.arch_type} + + def reset(self): + self._policy.reset() + return {} + + def select_action(self, observations: dict[str, Any]) -> np.ndarray: + images = [self.decode_image_from_list(img) for img in observations["image"]] + assert len(images) == 3, "Must provide exactly 3 images." + + state = torch.as_tensor( + observations["state"], dtype=torch.float32, device=self.device + ) + if state.ndim == 1: + state = state.unsqueeze(0) + if state.shape[1] < 24: + state = torch.cat([ + state, torch.zeros((1, 24 - state.shape[1]), device=self.device) + ], dim=1) + norm_state = self._normalizer.normalize_state(state).to(dtype=torch.float32) + + prompt = observations["prompt"] + image_mask = torch.tensor(observations["image_mask"], dtype=torch.int32, device=self.device) + action_mask = torch.tensor([observations["action_mask"]],dtype=torch.int32, device=self.device) + + autocast_context = ( + torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16) + if torch.cuda.is_available() and str(self.device).startswith("cuda") + else nullcontext() + ) + + with autocast_context: + action = self._policy.run_inference( + images=images, + image_mask=image_mask, + prompt=prompt, + state_input=norm_state, + action_mask=action_mask + ) + action = self._normalizer.denormalize_action(action.view(-1))[0] + return action.cpu().numpy() + + def decode_image_from_list( + self, img_list: list[np.ndarray] + ) -> torch.Tensor: + img_array = np.array(img_list, dtype=np.uint8) + img = cv2.resize(img_array, (448, 448)) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + pil = Image.fromarray(img) + return self.to_tensor(pil).to(self.device) + + @staticmethod + def load_policy_and_normalizer( + ckpt_dir: str, device: str | torch.device, n_action_steps: int | None = None + ) -> tuple[EVO1Policy, EVO1Normalizer]: + config = json.load(open(os.path.join(ckpt_dir, "config.json"))) + stats = json.load(open(os.path.join(ckpt_dir, "norm_stats.json"))) + + config["finetune_vlm"] = False + config["finetune_action_head"] = False + config["num_inference_timesteps"] = 32 + + policy = EVO1Policy(config, n_action_steps=n_action_steps).eval() + ckpt_path = os.path.join(ckpt_dir, "mp_rank_00_model_states.pt") + + checkpoint = torch.load( + ckpt_path, map_location="cpu", weights_only=False + ) + policy.load_state_dict(checkpoint["module"], strict=True) + policy = policy.to(device) + + normalizer = EVO1Normalizer(stats) + return policy, normalizer diff --git a/eval/pytorch_ref/policies/evo1/model/action_head/__init__.py b/eval/pytorch_ref/policies/evo1/model/action_head/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eval/pytorch_ref/policies/evo1/model/action_head/flow_matching.py b/eval/pytorch_ref/policies/evo1/model/action_head/flow_matching.py new file mode 100755 index 0000000..e5598f4 --- /dev/null +++ b/eval/pytorch_ref/policies/evo1/model/action_head/flow_matching.py @@ -0,0 +1,458 @@ +import math +import torch +import torch.nn as nn + + +class SinusoidalPositionalEncoding(nn.Module): + def __init__( + self, + dim: int, + max_len: int = 1000 + ) -> None: + super().__init__() + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + + def forward(self, seq_len: int) -> torch.Tensor: + if seq_len > self.pe.size(1): + self._extend_pe(seq_len) + return self.pe[:, :seq_len, :] + + def _extend_pe(self, new_max_len: int) -> None: + old_max_len, dim = self.pe.size(1), self.pe.size(2) + if new_max_len <= old_max_len: + return + extra_positions = torch.arange(old_max_len, new_max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, dim, 2, dtype=torch.float) * -(math.log(10000.0) / dim)) + extra_pe = torch.zeros(new_max_len - old_max_len, dim) + extra_pe[:, 0::2] = torch.sin(extra_positions * div_term) + extra_pe[:, 1::2] = torch.cos(extra_positions * div_term) + extra_pe = extra_pe.unsqueeze(0) + new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1) + self.pe = new_pe + + +class CategorySpecificLinear(nn.Module): + def __init__( + self, + in_dim: int, + out_dim: int, + num_categories: int = 1 + ) -> None: + super().__init__() + self.num_categories = num_categories + if num_categories <= 1: + self.linear = nn.Linear(in_dim, out_dim) + else: + self.weight = nn.Parameter(torch.randn(num_categories, in_dim, out_dim)) + self.bias = nn.Parameter(torch.randn(num_categories, out_dim)) + + def forward( + self, x: torch.Tensor, category_id: torch.LongTensor + ) -> torch.Tensor: + + if self.num_categories <= 1: + return self.linear(x) + + orig_shape = x.shape + x_flat = x.reshape(-1, orig_shape[-1]) + if category_id.dim() == 0: + + cid = category_id.item() + out = x_flat @ self.weight[cid] + self.bias[cid] + else: + + category_id = category_id.view(-1) + weight_selected = self.weight[category_id] + bias_selected = self.bias[category_id] + out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected + out_shape = orig_shape[:-1] + (out.shape[-1],) + return out.view(out_shape) + + +class CategorySpecificMLP(nn.Module): + def __init__( + self, + input_dim: int, + hidden_dim: int, + output_dim: int, + num_categories: int = 1 + ) -> None: + super().__init__() + self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories) + self.fc2 = CategorySpecificLinear(hidden_dim, output_dim, num_categories) + self.activation = nn.ReLU(inplace=True) + + def forward( + self, + x: torch.Tensor, + category_id: torch.LongTensor + ) -> torch.Tensor: + out = self.activation(self.fc1(x, category_id)) + out = self.fc2(out, category_id) + return out + + +class MultiEmbodimentActionEncoder(nn.Module): + def __init__( + self, + action_dim: int, + embed_dim: int, + hidden_dim: int, + horizon: int, + num_categories: int = 1 + ) -> None: + super().__init__() + self.horizon = horizon + self.embed_dim = embed_dim + self.num_categories = num_categories + + self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories) + self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories) + self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories) + + self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon) + self.activation = nn.ReLU(inplace=True) + + def forward( + self, + action_seq: torch.Tensor, + category_id: torch.LongTensor + ) -> torch.Tensor: + + B, H, D = action_seq.shape + assert H == self.horizon, "Action sequence length must match horizon" + + x = action_seq.reshape(B * H, D) + + if category_id.dim() == 0: + cat_ids = category_id.repeat(H * B) + else: + cat_ids = category_id.unsqueeze(1).repeat(1, H).reshape(B * H) + out = self.activation(self.W1(x, cat_ids)) + + pos_enc = self.pos_encoding(H).to(out.device) + pos_enc = pos_enc.repeat(B, 1, 1).reshape(B * H, -1) + out = out + pos_enc + out = self.activation(self.W2(out, cat_ids)) + out = self.W3(out, cat_ids) + out = out.view(B, H, self.embed_dim) + return out + + +class BasicTransformerBlock(nn.Module): + def __init__( + self, + embed_dim: int, + num_heads: int, + hidden_dim: int, + dropout: float = 0.0 + ) -> None: + super().__init__() + self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True) + self.norm1 = nn.LayerNorm(embed_dim) + self.norm2 = nn.LayerNorm(embed_dim) + self.ff = nn.Sequential( + nn.Linear(embed_dim, hidden_dim), + nn.GELU(), + nn.Linear(hidden_dim, embed_dim) + ) + + def forward( + self, + action_tokens: torch.Tensor, + context_tokens: torch.Tensor, + time_emb: torch.Tensor + ) -> torch.Tensor: + + x = self.norm1(action_tokens) + attn_out, _ = self.attn(x, context_tokens, context_tokens) + + x = action_tokens + attn_out + + x2 = self.norm2(x) + + if time_emb is not None: + x2 = x2 + time_emb.unsqueeze(1) + ff_out = self.ff(x2) + x = x + ff_out + return x + + +class FlowmatchingActionHead(nn.Module): + def __init__(self, + config: dict = None, + embed_dim: int = 896, + hidden_dim: int = 1024, + action_dim: int = 16*7, + horizon: int = 16, + per_action_dim: int = 7, + num_heads: int = 8, + num_layers: int = 8, + dropout: float = 0.0, + num_inference_timesteps: int = 20, + num_categories: int = 1 + ) -> None: + super().__init__() + + if config is not None: + embed_dim = getattr(config, "embed_dim", embed_dim) + hidden_dim = getattr(config, "hidden_dim", hidden_dim) + action_dim = getattr(config, "action_dim", action_dim) + horizon = getattr(config, "horizon", horizon) + num_heads = getattr(config, "num_heads", num_heads) + num_layers = getattr(config, "num_layers", num_layers) + dropout = getattr(config, "dropout", dropout) + num_inference_timesteps = getattr(config, "num_inference_timesteps", num_inference_timesteps) + num_categories = getattr(config, "num_categories", num_categories) + self.config = config + else: + from types import SimpleNamespace + self.config = SimpleNamespace( + embed_dim=embed_dim, hidden_dim=hidden_dim, + action_dim=action_dim, horizon=horizon, + num_heads=num_heads, num_layers=num_layers, + dropout=dropout, num_inference_timesteps=num_inference_timesteps, + num_categories=num_categories + ) + self.embed_dim = embed_dim + self.horizon = horizon + self.per_action_dim = config.per_action_dim + self.action_dim = config.action_dim + + + self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + embed_dim=embed_dim, num_heads=num_heads, + hidden_dim=embed_dim*4, dropout=dropout + ) + for _ in range(num_layers) + ]) + + self.norm_out = nn.LayerNorm(embed_dim) + self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim) + + self.mlp_head = CategorySpecificMLP( + input_dim=embed_dim, hidden_dim=hidden_dim, + output_dim=action_dim, num_categories=num_categories + ) + + self.state_encoder = None + if hasattr(self.config, "state_dim") and self.config.state_dim is not None: + + state_hidden = getattr(self.config, "state_hidden_dim", embed_dim) + + self.state_encoder = CategorySpecificMLP( + input_dim=self.config.state_dim, + hidden_dim=state_hidden, + output_dim=embed_dim, + num_categories=num_categories + ) + + self.action_encoder = None + if horizon > 1: + + per_action_dim = getattr(self.config, "per_action_dim", None) + if per_action_dim is None: + + per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim + self.action_encoder = MultiEmbodimentActionEncoder( + action_dim=per_action_dim, + embed_dim=embed_dim, + hidden_dim=embed_dim, + horizon=horizon, + num_categories=num_categories + ) + + def forward( + self, + fused_tokens: torch.Tensor, + state: torch.Tensor = None, + actions_gt: torch.Tensor = None, + embodiment_id: torch.LongTensor = None, + state_mask: torch.Tensor = None, + action_mask: torch.Tensor = None + ) -> tuple[torch.Tensor, torch.Tensor]: + + if actions_gt is None: + return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id) + B = fused_tokens.size(0) + device = fused_tokens.device + + if embodiment_id is None: + embodiment_id = torch.zeros(B, dtype=torch.long, device=device) + + context_tokens = fused_tokens + if state is not None and self.state_encoder is not None: + state_emb = self.state_encoder(state, embodiment_id) + state_emb = state_emb.unsqueeze(1) + + context_tokens = torch.cat([context_tokens, state_emb], dim=1) + + t = torch.distributions.Beta(2, 2).sample((B,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype) + time_index = (t * 1000).long() + time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0) + + actions_gt_seq = actions_gt + + noise = torch.rand_like(actions_gt) * 2 - 1 + + if action_mask is not None: + action_mask = action_mask.to(dtype=noise.dtype, device=noise.device) + assert action_mask.shape == noise.shape, f"action_mask shape {action_mask.shape} != noise shape {noise.shape}" + noise = noise * action_mask + + if self.horizon > 1: + noise_seq = noise.view(B, self.horizon, self.per_action_dim) + + else: + noise_seq = noise.unsqueeze(1) + + if self.horizon > 1: + t_broadcast = t.view(B, 1, 1) + else: + t_broadcast = t.view(B, 1) + action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq + + if self.horizon > 1 and self.action_encoder is not None: + + action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id) + else: + + if not hasattr(self, "single_action_proj"): + self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device) + action_tokens = self.single_action_proj(action_intermediate_seq) + + x = action_tokens + for block in self.transformer_blocks: + x = block(x, context_tokens, time_emb) + + x = self.norm_out(x) + + if self.horizon > 1: + + x_flat = x.reshape(B, -1) + + if not hasattr(self, "seq_pool_proj"): + + self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device) + x_pooled = self.seq_pool_proj(x_flat) + else: + + x_pooled = x.squeeze(1) + + pred_velocity = self.mlp_head(x_pooled, embodiment_id) + + return pred_velocity, noise + + def get_action( + self, + fused_tokens: torch.Tensor, + state: torch.Tensor = None, + embodiment_id: torch.LongTensor = None, + action_mask: torch.Tensor = None + ) -> torch.Tensor: + B = fused_tokens.size(0) + device = fused_tokens.device + if embodiment_id is None: + embodiment_id = torch.zeros(B, dtype=torch.long, device=device) + + context_tokens = fused_tokens + if state is not None and self.state_encoder is not None: + + state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1) + context_tokens = torch.cat([context_tokens, state_emb], dim=1) + + action_dim_total = getattr(self.config, "action_dim", None) + if action_dim_total is None: + + action_dim_total = self.action_dim + + if self.horizon > 1: + per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon) + else: + per_action_dim = action_dim_total + + action = (torch.rand(B, action_dim_total, device=device) * 2 - 1) + + if self.horizon > 1: + action_seq = action.view(B, self.horizon, per_action_dim) + + else: + action_seq = action.view(B, 1, per_action_dim) + + action_mask = action_mask.view(B, 1, per_action_dim).repeat(1,self.horizon,1) + + if action_mask is not None: + action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device) + assert action_mask.shape == action_seq.shape, f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}" + action_seq = action_seq * action_mask + else: + raise ValueError("action_mask must be provided for inference with flow matching.") + + N = int(getattr(self.config, "num_inference_timesteps", 32)) + dt = 1.0 / N + for i in range(N): + t = i / N + + time_index = int(t * 1000) + time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0) + time_emb = time_emb.unsqueeze(0).repeat(B, 1) + + + if self.horizon > 1 and self.action_encoder is not None: + + action_seq = action_seq * action_mask + action_tokens = self.action_encoder(action_seq, embodiment_id) + else: + if hasattr(self, "single_action_proj"): + action_tokens = self.single_action_proj(action_seq) + else: + + self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device) + action_tokens = self.single_action_proj(action_seq) + + x = action_tokens + for block in self.transformer_blocks: + x = block(x, context_tokens, time_emb) + x = self.norm_out(x) + + if self.horizon > 1: + x_flat = x.reshape(B, -1) + if hasattr(self, "seq_pool_proj"): + x_pooled = self.seq_pool_proj(x_flat) + else: + + self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device) + x_pooled = self.seq_pool_proj(x_flat) + else: + x_pooled = x.squeeze(1) + + pred = self.mlp_head(x_pooled, embodiment_id) + + action = action + dt * pred + + if self.horizon > 1: + action_seq = action.view(B, self.horizon, per_action_dim) + else: + action_seq = action.view(B, 1, per_action_dim) + + return action + + @property + def device(self) -> torch.device: + + return next(self.parameters()).device + + @property + def dtype(self) -> torch.dtype: + + return next(self.parameters()).dtype + diff --git a/eval/pytorch_ref/policies/evo1/model/internvl3/__init__.py b/eval/pytorch_ref/policies/evo1/model/internvl3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eval/pytorch_ref/policies/evo1/model/internvl3/internvl3_embedder.py b/eval/pytorch_ref/policies/evo1/model/internvl3/internvl3_embedder.py new file mode 100644 index 0000000..0151fa2 --- /dev/null +++ b/eval/pytorch_ref/policies/evo1/model/internvl3/internvl3_embedder.py @@ -0,0 +1,261 @@ +import torch +import torch.nn as nn +import torchvision.transforms as T +from torchvision.transforms.functional import InterpolationMode +from transformers import AutoModel, AutoTokenizer +from torchvision.transforms.functional import to_pil_image + +from PIL import Image +from typing import Union, List + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +# === Image Transformations === +def build_transform(input_size: int) -> T.Compose: + return T.Compose([ + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), + T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD) + ]) + + +# === Aspect Ratio Handling === +def find_closest_aspect_ratio( + aspect_ratio: float, + target_ratios: List[tuple[int, int]], + width: int, + height: int, + image_size: int +) -> tuple[int, int]: + best_ratio_diff = float('inf') + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_ar = ratio[0] / ratio[1] + diff = abs(aspect_ratio - target_ar) + if diff < best_ratio_diff: + best_ratio_diff = diff + best_ratio = ratio + elif diff == best_ratio_diff and area > 0.5 * image_size**2 * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + +def dynamic_preprocess( + image, min_num=1, max_num=1, image_size=448, use_thumbnail=False +) -> List[Image.Image]: + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + target_ratios = set( + (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if + i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size) + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size + ) + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +class InternVL3Embedder(nn.Module): + def __init__( + self, + model_name: str = "OpenGVLab/InternVL3-1B", + image_size: int = 448, + device: str = "cuda" + ) -> None: + super().__init__() + self.device = device + self.image_size = image_size + self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens + self.transform = build_transform(image_size) + self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False) + self.model = AutoModel.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + use_flash_attn=True, + low_cpu_mem_usage=True, + _fast_init=False, + ).to(self.device) + + if hasattr(self.model.language_model, 'model'): + layers = self.model.language_model.model.layers + + else: + layers = self.model.language_model.layers + layers = layers[:14] + + if hasattr(self.model.language_model, 'model'): + self.model.language_model.model.layers = torch.nn.ModuleList(layers) + else: + self.model.language_model.layers = torch.nn.ModuleList(layers) + self.model.language_model.lm_head = torch.nn.Identity() + + if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"): + self.model.vision_model.encoder.gradient_checkpointing = False + + + def _preprocess_images( + self, + image_tensors: List[Union[Image.Image, torch.Tensor]] + ) -> tuple[torch.Tensor, List[int]]: + + pixel_values_list = [] + for i, image in enumerate(image_tensors): + if isinstance(image, torch.Tensor): + image = to_pil_image(image) + tiles = dynamic_preprocess(image, image_size=self.image_size) + tile_tensors = torch.stack([self.transform(t) for t in tiles]) # (T_i, 3, 448, 448) + pixel_values_list.append(tile_tensors) + + pixel_values = torch.cat(pixel_values_list, dim=0).to(dtype=torch.bfloat16, device=self.device) + num_tiles_list = [pv.shape[0] for pv in pixel_values_list] + + return pixel_values, num_tiles_list + + def _build_multimodal_prompt( + self, + num_tiles_list: List[int], + text_prompt: str + ) -> str: + + prompt = '' + for i in range(len(num_tiles_list)): + prompt += f"Image-{i+1}: \n" + prompt += text_prompt.strip() + + IMG_CONTEXT_TOKEN = "" + IMG_START_TOKEN = "" + IMG_END_TOKEN = "" + + self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) + for tile_count in num_tiles_list: + token_count = self.model.num_image_token * tile_count + image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * token_count + IMG_END_TOKEN + prompt = prompt.replace("", image_tokens, 1) + + return prompt + + def _prepare_and_fuse_embeddings( + self, + prompt: str, + vit_embeds: torch.Tensor, + image_mask: torch.Tensor, + num_tiles_list: List[int] + ) -> tuple[torch.Tensor, torch.Tensor]: + + untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids + true_sequence_length = untruncated_ids.shape[1] + + if true_sequence_length > self.max_text_length: + print("\n" + "="*80) + print(f" WARNING: Input prompt was TRUNCATED!") + print(f" - Max Length Allowed : {self.max_text_length}") + print(f" - Actual Length : {true_sequence_length}") + print(f" - Truncated Prompt (first 100 chars): '{prompt[:100]}...'") + print("="*80 + "\n") + + model_inputs = self.tokenizer(prompt, return_tensors="pt", padding='max_length', truncation=True, max_length=self.max_text_length).to(self.device) + input_ids = model_inputs["input_ids"] + attention_mask = model_inputs["attention_mask"] + + + img_token_mask = (input_ids == self.img_context_token_id) + img_token_locations = torch.where(img_token_mask)[1] + input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone() + + B, N, C = input_embeds.shape + input_embeds = input_embeds.reshape(B * N, C) + input_ids = input_ids.reshape(B * N) + selected = (input_ids == self.img_context_token_id) + + try: + input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C) + except Exception as e: + vit_embeds = vit_embeds.reshape(-1, C) + print(f'warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, ' + f'vit_embeds.shape={vit_embeds.shape}') + n_token = selected.sum() + input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds[:n_token] + + tokens_per_tile = self.model.num_image_token + + torch.set_printoptions(profile="full", threshold=float('inf')) + torch.set_printoptions(profile="default") + current_token_idx = 0 + for i in range(len(image_mask)): + + num_tiles_for_this_image = num_tiles_list[i] + num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile + + if not image_mask[i]: + + start_idx = img_token_locations[current_token_idx] + end_idx = start_idx + num_tokens_for_this_image + + attention_mask[0, start_idx:end_idx] = 0 + + current_token_idx += num_tokens_for_this_image + + input_embeds = input_embeds.reshape(B, N, C) + + torch.set_printoptions(profile="full", threshold=float('inf')) + torch.set_printoptions(profile="default") + return input_embeds, attention_mask + + + def get_fused_image_text_embedding_from_tensor_images( + self, + image_tensors: list[Union[Image.Image, torch.Tensor]], + image_mask: torch.Tensor, + text_prompt: str, + return_cls_only: bool = True, + ) -> torch.Tensor: + pixel_values, num_tiles_list = self._preprocess_images(image_tensors) + if pixel_values.shape[0] == 0: + print("Warning: No valid images to process after masking.") + + import os as _os + _p = _os.environ.get("VLA_PROFILE") == "1" + if _p: + import time as _t + torch.cuda.synchronize(); _v0 = _t.perf_counter() + vit_embeds = self.model.extract_feature(pixel_values) + if _p: + torch.cuda.synchronize(); _v1 = _t.perf_counter() + fused_embeds = vit_embeds + prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt) + inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(prompt, fused_embeds, image_mask, num_tiles_list) + if _p: + torch.cuda.synchronize(); _v2 = _t.perf_counter() + print(f"[evo1-prof] vit={1000*(_v1-_v0):.1f}ms fuse={1000*(_v2-_v1):.1f}ms " + f"tiles={sum(num_tiles_list)}", flush=True) + + outputs = self.model.language_model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + output_hidden_states=True, + return_dict=True, + ) + fused_hidden = outputs.hidden_states[-1].to(torch.float32) + + return fused_hidden[:, 0, :] if return_cls_only else fused_hidden diff --git a/eval/pytorch_ref/policies/evo1/normalizer.py b/eval/pytorch_ref/policies/evo1/normalizer.py new file mode 100644 index 0000000..26f3b06 --- /dev/null +++ b/eval/pytorch_ref/policies/evo1/normalizer.py @@ -0,0 +1,49 @@ +import json +import torch + + +class EVO1Normalizer: + def __init__(self, stats_or_path: str) -> None: + if isinstance(stats_or_path, str): + with open(stats_or_path, "r") as f: + stats = json.load(f) + else: + stats = stats_or_path + + def pad_to_24(x) -> torch.Tensor: + x = torch.tensor(x, dtype=torch.float32) + if x.shape[0] < 24: + pad = torch.zeros(24 - x.shape[0], dtype=torch.float32) + x = torch.cat([x, pad], dim=0) + elif x.shape[0] > 24: + raise ValueError(f"Input length {x.shape[0]} exceeds expected 24") + return x + + if len(stats) != 1: + raise ValueError(f"norm_stats.json should contain only one robot key, but: {list(stats.keys())}") + + robot_key = list(stats.keys())[0] + robot_stats = stats[robot_key] + + self.state_min = pad_to_24(robot_stats["observation.state"]["min"]) + self.state_max = pad_to_24(robot_stats["observation.state"]["max"]) + self.action_min = pad_to_24(robot_stats["action"]["min"]) + self.action_max = pad_to_24(robot_stats["action"]["max"]) + + + def normalize_state( + self, state: torch.Tensor + ) -> torch.Tensor: + state_min = self.state_min.to(state.device, dtype=state.dtype) + state_max = self.state_max.to(state.device, dtype=state.dtype) + return torch.clamp(2 * (state - state_min) / (state_max - state_min + 1e-8) - 1, -1.0, 1.0) + + + def denormalize_action( + self, action: torch.Tensor + ) -> torch.Tensor: + action_min = self.action_min.to(action.device, dtype=action.dtype) + action_max = self.action_max.to(action.device, dtype=action.dtype) + if action.ndim == 1: + action = action.view(1, -1) + return (action + 1.0) / 2.0 * (action_max - action_min + 1e-8) + action_min diff --git a/eval/pytorch_ref/policies/evo1/policy.py b/eval/pytorch_ref/policies/evo1/policy.py new file mode 100644 index 0000000..be1bff4 --- /dev/null +++ b/eval/pytorch_ref/policies/evo1/policy.py @@ -0,0 +1,182 @@ +import os +from types import SimpleNamespace +from typing import List, Union, Tuple +from PIL import Image + +import torch +import torch.nn as nn +from collections import deque + +from policies.evo1.model.internvl3.internvl3_embedder import InternVL3Embedder +from policies.evo1.model.action_head.flow_matching import FlowmatchingActionHead + + +class EVO1Policy(nn.Module): + def __init__(self, config: dict, n_action_steps: int = None): + super().__init__() + self.config = config + self._device = config.get("device", "cuda") + self.return_cls_only = config.get("return_cls_only", False) + vlm_name = config.get("vlm_name", "OpenGVLab/InternVL3-1B") + self.embedder = InternVL3Embedder(model_name=vlm_name, device=self._device) + + action_head_type = config.get("action_head", "flowmatching").lower() + + if action_head_type == "flowmatching": + + horizon = config.get("action_horizon", config.get("horizon", 16)) + per_action_dim = config.get("per_action_dim", 7) + action_dim = horizon * per_action_dim + + config["horizon"] = horizon + config["per_action_dim"] = per_action_dim + config["action_dim"] = action_dim + + if action_dim != horizon * per_action_dim: + raise ValueError(f"action_dim ({action_dim}) ≠ horizon ({horizon}) × per_action_dim ({per_action_dim})") + + self.horizon = horizon + self.per_action_dim = per_action_dim + + self.action_head = FlowmatchingActionHead(config=SimpleNamespace( + embed_dim=config.get("embed_dim", 896), + hidden_dim=config.get("hidden_dim", 1024), + action_dim=action_dim, + horizon=horizon, + per_action_dim=per_action_dim, + state_dim=config.get("state_dim", 7), + state_hidden_dim=config.get("state_hidden_dim", 1024), + num_heads=config.get("num_heads", 8), + num_layers=config.get("num_layers", 8), + dropout=config.get("dropout", 0.0), + num_inference_timesteps=config.get("num_inference_timesteps", 50), + num_categories=config.get("num_categories", 1) + )).to(self._device) + else: + raise NotImplementedError(f"Unknown action_head: {action_head_type}") + + self.n_action_steps = n_action_steps or config.get("n_action_steps", 14) + assert self.n_action_steps <= self.horizon, "n_action_steps must be less than or equal to the horizon length." + self.reset() + + def reset(self): + """This should be called whenever the environment is reset.""" + self._queues = deque(maxlen=self.horizon) + + def get_vl_embeddings( + self, + images: List[Image.Image], + image_mask: torch.Tensor, + prompt: str = "", + return_cls_only: Union[bool, None] = None + ) -> torch.Tensor: + if return_cls_only is None: + return_cls_only = self.return_cls_only + + if images is None or len(images) == 0: + raise ValueError("Must provide at least one image (PIL.Image). Got `images=None` or empty list.") + return self.embedder.get_fused_image_text_embedding_from_tensor_images( + image_tensors=images, + image_mask=image_mask, + text_prompt=prompt, + return_cls_only=return_cls_only, + ) + + def prepare_state(self, state_input: Union[list, torch.Tensor]) -> torch.Tensor: + if isinstance(state_input, list): + state_tensor = torch.tensor(state_input) + elif isinstance(state_input, torch.Tensor): + state_tensor = state_input + else: + raise TypeError("Unsupported state input type") + + if state_tensor.ndim == 1: + state_tensor = state_tensor.unsqueeze(0) + + return state_tensor.to(self._device) + + + def predict_action( + self, + fused_tokens: torch.Tensor, + state: torch.Tensor, + actions_gt: torch.Tensor = None, + action_mask: torch.Tensor = None, + embodiment_ids: torch.Tensor = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if actions_gt is None: + return self.action_head.get_action(fused_tokens, state=state, action_mask=action_mask, embodiment_id=embodiment_ids) + else: + return self.action_head(fused_tokens, state=state, actions_gt=actions_gt, action_mask=action_mask, embodiment_id=embodiment_ids) + + def forward( + self, + fused_tokens: torch.Tensor, + state: torch.Tensor = None, + actions_gt: torch.Tensor = None, + action_mask: torch.Tensor = None, + embodiment_ids: torch.Tensor = None + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + return self.predict_action(fused_tokens, state, actions_gt, action_mask, embodiment_ids) + + @torch.no_grad() + def run_inference( + self, + images: List[Union[Image.Image, torch.Tensor]], + image_mask: torch.Tensor, + prompt: str, + state_input: Union[list, torch.Tensor], + return_cls_only: Union[bool, None] = None, + action_mask: Union[torch.Tensor, None] = None + ) -> torch.Tensor: + if not self._check_get_actions_condition(): + return self._queues.popleft() + # VLA_PROFILE=1 splits the forward into its two hot stages (InternVL3 + # embedder vs flow-matching action head) so the PyTorch reference can be + # compared against vla.cpp's own vision/inference breakdown. CUDA is + # async, so each stage is synchronized before it is timed. + _prof = os.environ.get("VLA_PROFILE") == "1" + if _prof: + import time as _t + torch.cuda.synchronize(); _t0 = _t.perf_counter() + fused_tokens = self.get_vl_embeddings( + images=images, + image_mask=image_mask, + prompt=prompt, + return_cls_only=return_cls_only + ) + if _prof: + torch.cuda.synchronize(); _t1 = _t.perf_counter() + state_tensor = self.prepare_state(state_input) + actions = self.predict_action(fused_tokens, state_tensor, action_mask=action_mask) + if _prof: + torch.cuda.synchronize(); _t2 = _t.perf_counter() + print(f"[evo1-prof] embedder={1000*(_t1-_t0):.1f}ms " + f"action_head={1000*(_t2-_t1):.1f}ms", flush=True) + actions = actions.reshape(-1, self.horizon, self.per_action_dim) + self._queues.extend(actions.transpose(0, 1)) + return self._queues.popleft() + + def _check_get_actions_condition(self) -> bool: + action_length = len(self._queues) + return ( + (action_length == (self.horizon - self.n_action_steps)) or \ + (action_length == 0) + ) + + def _freeze_module(self, module: nn.Module, name: str) -> None: + print(f"Freezing {name} parameters...") + for p in module.parameters(): + p.requires_grad = False + + def set_finetune_flags(self) -> None: + config = self.config + if not config.get("finetune_vlm", False): + self._freeze_module(self.embedder, "VLM (InternVL3)") + else: + print("Finetuning VLM (InternVL3)...") + + if not config.get("finetune_action_head", False): + self._freeze_module(self.action_head, "Action Head") + else: + print("Finetuning Action Head...") diff --git a/eval/pytorch_ref/policies/gr00t_n15/__init__.py b/eval/pytorch_ref/policies/gr00t_n15/__init__.py new file mode 100644 index 0000000..c016984 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/__init__.py @@ -0,0 +1,125 @@ +from typing import Any +import statistics +import time + +import torch +import numpy as np + +from policies.gr00t_n15.groot.modeling_groot import GrootPolicy +from policies.torch_compile import maybe_compile +from lerobot.configs.policies import PreTrainedConfig +from lerobot.policies.factory import make_pre_post_processors + + +class GR00TN15PolicyPipeline: + arch_type: str = "gr00t-n15" + + def __init__( + self, + model_id: str = "liorbenhorin-nv/groot-libero_object-64_40000", + device: torch.device | str = None, + n_action_steps: int | None = None, + ) -> None: + if device is None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model_id = model_id + self.device = device + self._pre_ms: list[float] = [] + self._fwd_ms: list[float] = [] + self._post_ms: list[float] = [] + + # The 2.4B-param model loads in fp32 by default (~9.6 GB) and lerobot's + # from_pretrained moves it straight to config.device — that OOMs an 8 GB + # GPU. Load on CPU, then cast to bf16 (this model already computes under + # torch.autocast(bf16), so bf16-resident weights are the same precision + # at ~4.8 GB) and move to the target device. Pinning the config to "cpu" + # also avoids the transient full-checkpoint-on-GPU spike during loading. + cfg = PreTrainedConfig.from_pretrained(model_id) + cfg.device = "cpu" + self._policy = GrootPolicy.from_pretrained( + model_id, config=cfg, strict=False + ).eval() + self._policy = self._policy.to(dtype=torch.bfloat16, device=device) + self._policy.config.device = str(device) + + # Chunk replay. lerobot policies queue `config.n_action_steps` actions + # from each predicted chunk and pop one per `select_action`, which is + # exactly vla.cpp's client-side --n-action-steps. Override it so both + # stacks run the same control cadence. + if n_action_steps is not None: + print( + f"[gr00t_n15] n_action_steps: {self._policy.config.n_action_steps} -> {n_action_steps}", + flush=True, + ) + self._policy.config.n_action_steps = n_action_steps + # GrootPolicy.__init__ already built _action_queue as + # deque(maxlen=config.n_action_steps) with the checkpoint's value, + # and its select_action extends that deque with the *full* horizon + # (unlike lerobot's pi0, which slices to n_action_steps first). So + # the override alone does not take effect — the stale maxlen keeps + # queueing actions and most select_action calls become cheap pops. + # reset() rebuilds the deque against the value we just set. + self._policy.reset() + print( + f"[gr00t_n15] action queue maxlen: {self._policy._action_queue.maxlen}", + flush=True, + ) + + self._preprocess, self._postprocess = make_pre_post_processors( + self._policy.config, model_id, + preprocessor_overrides={"device_processor": {"device": str(device)}}, + ) + + # Optional torch.compile of the Eagle backbone + flow-matching head. + # GrootPolicy is a thin lerobot wrapper (queueing, normalization) around + # _groot_model, which is where every FLOP is spent — so that submodule + # is the compile target, not the wrapper. + self._policy._groot_model = maybe_compile( + self._policy._groot_model, tag="gr00t_n15._groot_model", + ) + + def reset(self): + self._policy.reset() + return {} + + def select_action(self, observations: dict[str, Any]) -> np.ndarray: + # Phase split, printed periodically below. No CUDA syncs here on + # purpose: adding them would serialize work that normally overlaps and + # inflate the total this is meant to explain. `pre` is pure host work + # so it is exact; `fwd` only counts kernel launches, and the GPU wait + # lands in `post`, whose first act is a device->host copy. + t0 = time.perf_counter() + + # Convert any numpy arrays in observations to torch tensors + for key in observations: + if isinstance(observations[key], np.ndarray): + observations[key] = torch.from_numpy(observations[key]) + + # Run the policy + batch = self._preprocess(observations) + t1 = time.perf_counter() + with torch.inference_mode(): + pred_action = self._policy.select_action(batch) + t2 = time.perf_counter() + pred_action = self._postprocess(pred_action) + + # Return the action as a numpy array + out = pred_action[0].cpu().numpy() + t3 = time.perf_counter() + + self._pre_ms.append(1000.0 * (t1 - t0)) + self._fwd_ms.append(1000.0 * (t2 - t1)) + self._post_ms.append(1000.0 * (t3 - t2)) + n = len(self._pre_ms) + if n % 50 == 0: + def q(xs): + s = sorted(xs) + return (statistics.fmean(s), s[len(s) // 2], s[int(0.95 * (len(s) - 1))], s[0], s[-1]) + for name, xs in (("pre", self._pre_ms), ("fwd", self._fwd_ms), ("post", self._post_ms)): + m, md, p95, lo, hi = q(xs[-50:]) + print(f"[gr00t_n15] call={n} {name:4s} mean={m:7.2f} med={md:7.2f} " + f"p95={p95:7.2f} min={lo:7.2f} max={hi:7.2f}", flush=True) + return out + + def get_arch(self) -> dict[str, str]: + return {"arch": self.arch_type} diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/__init__.py b/eval/pytorch_ref/policies/gr00t_n15/groot/__init__.py new file mode 100644 index 0000000..ddc0a35 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/__init__.py @@ -0,0 +1,4 @@ +from .configuration_groot import GrootConfig +from .modeling_groot import GrootPolicy + +__all__ = ["GrootConfig", "GrootPolicy"] diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/__init__.py b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/__init__.py new file mode 100644 index 0000000..3159bfe --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/action_encoder.py b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/action_encoder.py new file mode 100644 index 0000000..c6fa0a7 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/action_encoder.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + + +def swish(x): + return x * torch.sigmoid(x) + + +class SinusoidalPositionalEncoding(nn.Module): + """ + Produces a sinusoidal encoding of shape (B, T, w) + given timesteps of shape (B, T). + """ + + def __init__(self, embedding_dim): + super().__init__() + self.embedding_dim = embedding_dim + + def forward(self, timesteps): + # timesteps: shape (B, T) + # We'll compute sin/cos frequencies across dim T + timesteps = timesteps.float() # ensure float + + b, t = timesteps.shape + device = timesteps.device + + half_dim = self.embedding_dim // 2 + # typical log space frequencies for sinusoidal encoding + exponent = -torch.arange(half_dim, dtype=torch.float, device=device) * ( + torch.log(torch.tensor(10000.0)) / half_dim + ) + # Expand timesteps to (B, T, 1) then multiply + freqs = timesteps.unsqueeze(-1) * exponent.exp() # (B, T, half_dim) + + sin = torch.sin(freqs) + cos = torch.cos(freqs) + enc = torch.cat([sin, cos], dim=-1) # (B, T, w) + + return enc diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/cross_attention_dit.py b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/cross_attention_dit.py new file mode 100644 index 0000000..40f7ba6 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/cross_attention_dit.py @@ -0,0 +1,370 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torch.nn.functional as F # noqa: N812 +from diffusers import ConfigMixin, ModelMixin +from diffusers.configuration_utils import register_to_config +from diffusers.models.attention import Attention, FeedForward +from diffusers.models.embeddings import ( + SinusoidalPositionalEmbedding, + TimestepEmbedding, + Timesteps, +) +from torch import nn + + +class TimestepEncoder(nn.Module): + def __init__(self, embedding_dim, compute_dtype=torch.float32): + super().__init__() + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + + def forward(self, timesteps): + dtype = next(self.parameters()).dtype + timesteps_proj = self.time_proj(timesteps).to(dtype) + timesteps_emb = self.timestep_embedder(timesteps_proj) # (N, D) + return timesteps_emb + + +class AdaLayerNorm(nn.Module): + def __init__( + self, + embedding_dim: int, + norm_elementwise_affine: bool = False, + norm_eps: float = 1e-5, + chunk_dim: int = 0, + ): + super().__init__() + self.chunk_dim = chunk_dim + output_dim = embedding_dim * 2 + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, output_dim) + self.norm = nn.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine) + + def forward( + self, + x: torch.Tensor, + temb: torch.Tensor | None = None, + ) -> torch.Tensor: + temb = self.linear(self.silu(temb)) + scale, shift = temb.chunk(2, dim=1) + x = self.norm(x) * (1 + scale[:, None]) + shift[:, None] + return x + + +class BasicTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: int | None = None, + activation_fn: str = "geglu", + attention_bias: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'ada_norm_continuous', 'layer_norm_i2vgen' + norm_eps: float = 1e-5, + final_dropout: bool = False, + attention_type: str = "default", + positional_embeddings: str | None = None, + num_positional_embeddings: int | None = None, + ff_inner_dim: int | None = None, + ff_bias: bool = True, + attention_out_bias: bool = True, + ): + super().__init__() + self.dim = dim + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + self.dropout = dropout + self.cross_attention_dim = cross_attention_dim + self.activation_fn = activation_fn + self.attention_bias = attention_bias + self.norm_elementwise_affine = norm_elementwise_affine + self.positional_embeddings = positional_embeddings + self.num_positional_embeddings = num_positional_embeddings + self.norm_type = norm_type + + if positional_embeddings and (num_positional_embeddings is None): + raise ValueError( + "If `positional_embeddings` type is defined, `num_positional_embeddings` must also be defined." + ) + + if positional_embeddings == "sinusoidal": + self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings) + else: + self.pos_embed = None + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + if norm_type == "ada_norm": + self.norm1 = AdaLayerNorm(dim) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps) + + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) + + # 3. Feed-forward + self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout, + inner_dim=ff_inner_dim, + bias=ff_bias, + ) + if final_dropout: + self.final_dropout = nn.Dropout(dropout) + else: + self.final_dropout = None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + encoder_hidden_states: torch.Tensor | None = None, + encoder_attention_mask: torch.Tensor | None = None, + temb: torch.LongTensor | None = None, + ) -> torch.Tensor: + # 0. Self-Attention + if self.norm_type == "ada_norm": + norm_hidden_states = self.norm1(hidden_states, temb) + else: + norm_hidden_states = self.norm1(hidden_states) + + if self.pos_embed is not None: + norm_hidden_states = self.pos_embed(norm_hidden_states) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + # encoder_attention_mask=encoder_attention_mask, + ) + if self.final_dropout: + attn_output = self.final_dropout(attn_output) + + hidden_states = attn_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + # 4. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + ff_output = self.ff(norm_hidden_states) + + hidden_states = ff_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + return hidden_states + + +class DiT(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + num_attention_heads: int = 8, + attention_head_dim: int = 64, + output_dim: int = 26, + num_layers: int = 12, + dropout: float = 0.1, + attention_bias: bool = True, + activation_fn: str = "gelu-approximate", + num_embeds_ada_norm: int | None = 1000, + upcast_attention: bool = False, + norm_type: str = "ada_norm", + norm_elementwise_affine: bool = False, + norm_eps: float = 1e-5, + max_num_positional_embeddings: int = 512, + compute_dtype=torch.float32, + final_dropout: bool = True, + positional_embeddings: str | None = "sinusoidal", + interleave_self_attention=False, + cross_attention_dim: int | None = None, + ): + super().__init__() + + self.attention_head_dim = attention_head_dim + self.inner_dim = self.config.num_attention_heads * self.config.attention_head_dim + self.gradient_checkpointing = False + + # Timestep encoder + self.timestep_encoder = TimestepEncoder( + embedding_dim=self.inner_dim, compute_dtype=self.config.compute_dtype + ) + + all_blocks = [] + for idx in range(self.config.num_layers): + use_self_attn = idx % 2 == 1 and interleave_self_attention + curr_cross_attention_dim = cross_attention_dim if not use_self_attn else None + + all_blocks += [ + BasicTransformerBlock( + self.inner_dim, + self.config.num_attention_heads, + self.config.attention_head_dim, + dropout=self.config.dropout, + activation_fn=self.config.activation_fn, + attention_bias=self.config.attention_bias, + upcast_attention=self.config.upcast_attention, + norm_type=norm_type, + norm_elementwise_affine=self.config.norm_elementwise_affine, + norm_eps=self.config.norm_eps, + positional_embeddings=positional_embeddings, + num_positional_embeddings=self.config.max_num_positional_embeddings, + final_dropout=final_dropout, + cross_attention_dim=curr_cross_attention_dim, + ) + ] + self.transformer_blocks = nn.ModuleList(all_blocks) + + # Output blocks + self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim) + self.proj_out_2 = nn.Linear(self.inner_dim, self.config.output_dim) + print( + "Total number of DiT parameters: ", + sum(p.numel() for p in self.parameters() if p.requires_grad), + ) + + def forward( + self, + hidden_states: torch.Tensor, # Shape: (B, T, D) + encoder_hidden_states: torch.Tensor, # Shape: (B, S, D) + timestep: torch.LongTensor | None = None, + encoder_attention_mask: torch.Tensor | None = None, + return_all_hidden_states: bool = False, + ): + # Encode timesteps + temb = self.timestep_encoder(timestep) + + # Process through transformer blocks - single pass through the blocks + hidden_states = hidden_states.contiguous() + encoder_hidden_states = encoder_hidden_states.contiguous() + + all_hidden_states = [hidden_states] + + # Process through transformer blocks + for idx, block in enumerate(self.transformer_blocks): + if idx % 2 == 1 and self.config.interleave_self_attention: + hidden_states = block( + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + temb=temb, + ) + else: + hidden_states = block( + hidden_states, + attention_mask=None, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=None, + temb=temb, + ) + all_hidden_states.append(hidden_states) + + # Output processing + conditioning = temb + shift, scale = self.proj_out_1(F.silu(conditioning)).chunk(2, dim=1) + hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None] + if return_all_hidden_states: + return self.proj_out_2(hidden_states), all_hidden_states + else: + return self.proj_out_2(hidden_states) + + +class SelfAttentionTransformer(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + num_attention_heads: int = 8, + attention_head_dim: int = 64, + output_dim: int = 26, + num_layers: int = 12, + dropout: float = 0.1, + attention_bias: bool = True, + activation_fn: str = "gelu-approximate", + num_embeds_ada_norm: int | None = 1000, + upcast_attention: bool = False, + max_num_positional_embeddings: int = 512, + compute_dtype=torch.float32, + final_dropout: bool = True, + positional_embeddings: str | None = "sinusoidal", + interleave_self_attention=False, + ): + super().__init__() + + self.attention_head_dim = attention_head_dim + self.inner_dim = self.config.num_attention_heads * self.config.attention_head_dim + self.gradient_checkpointing = False + + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + self.inner_dim, + self.config.num_attention_heads, + self.config.attention_head_dim, + dropout=self.config.dropout, + activation_fn=self.config.activation_fn, + attention_bias=self.config.attention_bias, + upcast_attention=self.config.upcast_attention, + positional_embeddings=positional_embeddings, + num_positional_embeddings=self.config.max_num_positional_embeddings, + final_dropout=final_dropout, + ) + for _ in range(self.config.num_layers) + ] + ) + print( + "Total number of SelfAttentionTransformer parameters: ", + sum(p.numel() for p in self.parameters() if p.requires_grad), + ) + + def forward( + self, + hidden_states: torch.Tensor, # Shape: (B, T, D) + return_all_hidden_states: bool = False, + ): + # Process through transformer blocks - single pass through the blocks + hidden_states = hidden_states.contiguous() + all_hidden_states = [hidden_states] + + # Process through transformer blocks + for _idx, block in enumerate(self.transformer_blocks): + hidden_states = block(hidden_states) + all_hidden_states.append(hidden_states) + + if return_all_hidden_states: + return hidden_states, all_hidden_states + else: + return hidden_states diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/flow_matching_action_head.py b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/flow_matching_action_head.py new file mode 100644 index 0000000..bfc456b --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/action_head/flow_matching_action_head.py @@ -0,0 +1,406 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import torch +import torch.nn.functional as F # noqa: N812 +from torch import nn +from torch.distributions import Beta + +from lerobot.utils.import_utils import _transformers_available + +# Conditional import for type checking and lazy loading +if TYPE_CHECKING or _transformers_available: + from transformers import PretrainedConfig + from transformers.feature_extraction_utils import BatchFeature +else: + PretrainedConfig = object + BatchFeature = None + +from lerobot.policies.groot.action_head.action_encoder import ( + SinusoidalPositionalEncoding, + swish, +) + +from .cross_attention_dit import DiT, SelfAttentionTransformer + + +class CategorySpecificLinear(nn.Module): + def __init__(self, num_categories, input_dim, hidden_dim): + super().__init__() + self.num_categories = num_categories + # For each category, we have separate weights and biases. + self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, hidden_dim)) + self.b = nn.Parameter(torch.zeros(num_categories, hidden_dim)) + + def forward(self, x, cat_ids): + selected_w = self.W[cat_ids] + selected_b = self.b[cat_ids] + return torch.bmm(x, selected_w) + selected_b.unsqueeze(1) + + +class CategorySpecificMLP(nn.Module): + def __init__(self, num_categories, input_dim, hidden_dim, output_dim): + super().__init__() + self.num_categories = num_categories + self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim) + self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim) + + def forward(self, x, cat_ids): + hidden = F.relu(self.layer1(x, cat_ids)) + return self.layer2(hidden, cat_ids) + + +class MultiEmbodimentActionEncoder(nn.Module): + def __init__(self, action_dim, hidden_size, num_embodiments): + super().__init__() + self.hidden_size = hidden_size + self.num_embodiments = num_embodiments + + # W1: R^{w x d}, W2: R^{w x 2w}, W3: R^{w x w} + self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size) # (d -> w) + self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size) # (2w -> w) + self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size) # (w -> w) + self.pos_encoding = SinusoidalPositionalEncoding(hidden_size) + + def forward(self, actions, timesteps, cat_ids): + """ + actions: shape (B, T, action_dim) + timesteps: shape (B,) -- a single scalar per batch item + cat_ids: shape (B,) + returns: shape (B, T, hidden_size) + """ + b, t, _ = actions.shape + + # 1) Expand each batch's single scalar time 'tau' across all T steps + # so that shape => (B, T) + # e.g. if timesteps is (B,), replicate across T + if timesteps.dim() == 1 and timesteps.shape[0] == b: + # shape (B,) => (B,T) + timesteps = timesteps.unsqueeze(1).expand(-1, t) + else: + raise ValueError("Expected `timesteps` to have shape (B,) so we can replicate across T.") + + # 2) Standard action MLP step for shape => (B, T, w) + a_emb = self.W1(actions, cat_ids) + + # 3) Get the sinusoidal encoding (B, T, w) + tau_emb = self.pos_encoding(timesteps).to(dtype=a_emb.dtype) + + # 4) Concat along last dim => (B, T, 2w), then W2 => (B, T, w), swish + x = torch.cat([a_emb, tau_emb], dim=-1) + x = swish(self.W2(x, cat_ids)) + + # 5) Finally W3 => (B, T, w) + x = self.W3(x, cat_ids) + return x + + +@dataclass +class FlowmatchingActionHeadConfig(PretrainedConfig): + """NOTE: N1.5 uses XEmbFlowmatchingPolicyHeadConfig as action head""" + + add_pos_embed: bool = field(default=True, metadata={"help": "Whether to add positional embedding"}) + model_dtype: str = field(default="float32", metadata={"help": "Model data type."}) + diffusion_model_cfg: dict = field(default=None, metadata={"help": "Diffusion model configuration."}) + input_embedding_dim: int = field(default=1536, metadata={"help": "Input embedding channel dimension."}) + backbone_embedding_dim: int = field( + default=1536, metadata={"help": "Backbone embedding channel dimension."} + ) + + hidden_size: int = field(default=1024, metadata={"help": "Input embedding dimension."}) + max_seq_len: int = field(default=1024, metadata={"help": "Maximum Sequence Length"}) + action_dim: int = field(default=None, metadata={"help": "Action dimension."}) + action_horizon: int = field(default=None, metadata={"help": "Action horizon."}) + noise_beta_alpha: float = field(default=1.5, metadata={"help": ""}) + noise_beta_beta: float = field(default=1.0, metadata={"help": ""}) + noise_s: float = field(default=0.999, metadata={"help": "Flow matching noise Beta distribution s."}) + num_timestep_buckets: int = field( + default=1000, metadata={"help": "Number of timestep discretization buckets."} + ) + num_inference_timesteps: int = field( + default=None, + metadata={"help": "Number of inference steps for noise diffusion."}, + ) + max_num_embodiments: int = field(default=32, metadata={"help": "Number of embodiments."}) + tune_projector: bool = field(default=True, metadata={"help": "Whether to tune the projector."}) + tune_diffusion_model: bool = field( + default=True, metadata={"help": "Whether to tune the diffusion model."} + ) + load_pretrained_det_decode_layer_path: str = field( + default=None, metadata={"help": "Path to pretrained detection model."} + ) + detection_coeff: float = field(default=1.0, metadata={"help": "Detection coefficient."}) + + freeze_decode_layer: bool = field(default=False) + expand_batch: int = field(default=None) + use_vlln: bool = field(default=True) + + vl_self_attention_cfg: dict = field(default=None) + num_target_vision_tokens: int = field(default=32, metadata={"help": "Number of target vision tokens."}) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + for key, value in kwargs.items(): + setattr(self, key, value) + + +class FlowmatchingActionHead(nn.Module): + config_class = FlowmatchingActionHeadConfig + supports_gradient_checkpointing = True + + def __init__( + self, + config: FlowmatchingActionHeadConfig, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.input_embedding_dim = config.input_embedding_dim + + self.model = DiT(**config.diffusion_model_cfg) + self.action_dim = config.action_dim + self.action_horizon = config.action_horizon + self.num_inference_timesteps = config.num_inference_timesteps + + self.state_encoder = CategorySpecificMLP( + num_categories=config.max_num_embodiments, + input_dim=config.max_state_dim, + hidden_dim=self.hidden_size, + output_dim=self.input_embedding_dim, + ) + self.action_encoder = MultiEmbodimentActionEncoder( + action_dim=config.action_dim, + hidden_size=self.input_embedding_dim, + num_embodiments=config.max_num_embodiments, + ) + self.action_decoder = CategorySpecificMLP( + num_categories=config.max_num_embodiments, + input_dim=self.hidden_size, + hidden_dim=self.hidden_size, + output_dim=self.action_dim, + ) + self.future_tokens = nn.Embedding(config.num_target_vision_tokens, self.input_embedding_dim) + nn.init.normal_(self.future_tokens.weight, mean=0.0, std=0.02) + + self.vlln = nn.LayerNorm(config.backbone_embedding_dim) if config.use_vlln else nn.Identity() + self.vl_self_attention = ( + SelfAttentionTransformer(**config.vl_self_attention_cfg) if config.use_vlln else nn.Identity() + ) + + if config.add_pos_embed: + self.position_embedding = nn.Embedding(config.max_seq_len, self.input_embedding_dim) + nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02) + + self.beta_dist = Beta(config.noise_beta_alpha, config.noise_beta_beta) + self.num_timestep_buckets = config.num_timestep_buckets + self.config = config + self.set_trainable_parameters(config.tune_projector, config.tune_diffusion_model) + + def set_trainable_parameters(self, tune_projector: bool, tune_diffusion_model: bool): + self.tune_projector = tune_projector + self.tune_diffusion_model = tune_diffusion_model + for p in self.parameters(): + p.requires_grad = True + if not tune_projector: + self.state_encoder.requires_grad_(False) + self.action_encoder.requires_grad_(False) + self.action_decoder.requires_grad_(False) + if self.config.add_pos_embed: + self.position_embedding.requires_grad_(False) + if not tune_diffusion_model: + self.model.requires_grad_(False) + print(f"Tune action head projector: {self.tune_projector}") + print(f"Tune action head diffusion model: {self.tune_diffusion_model}") + # Check if any parameters are still trainable. If not, print a warning. + if not tune_projector and not tune_diffusion_model: + for name, p in self.named_parameters(): + if p.requires_grad: + print(f"Action head trainable parameter: {name}") + if not any(p.requires_grad for p in self.parameters()): + print("Warning: No action head trainable parameters found.") + + def set_frozen_modules_to_eval_mode(self): + """ + Huggingface will call model.train() at each training_step. To ensure + the expected behaviors for modules like dropout, batchnorm, etc., we + need to call model.eval() for the frozen modules. + """ + if self.training: + if not self.tune_projector: + self.state_encoder.eval() + self.action_encoder.eval() + self.action_decoder.eval() + if self.config.add_pos_embed: + self.position_embedding.eval() + if not self.tune_diffusion_model: + self.model.eval() + + def sample_time(self, batch_size, device, dtype): + sample = self.beta_dist.sample([batch_size]).to(device, dtype=dtype) + return (self.config.noise_s - sample) / self.config.noise_s + + def prepare_input(self, batch: dict) -> BatchFeature: + return BatchFeature(data=batch) + + def process_backbone_output(self, backbone_output: BatchFeature) -> BatchFeature: + backbone_features = backbone_output["backbone_features"] + backbone_features = self.vlln(backbone_features) + backbone_features = self.vl_self_attention(backbone_features) + backbone_output["backbone_features"] = backbone_features + return backbone_output + + def forward(self, backbone_output: BatchFeature, action_input: BatchFeature) -> BatchFeature: + # Set frozen modules to eval + self.set_frozen_modules_to_eval_mode() + + backbone_output = self.process_backbone_output(backbone_output) + + if self.config.expand_batch is not None: + for k, v in backbone_output.items(): + ndim = len(v.shape) + factors = [self.config.expand_batch] + while len(factors) < ndim: + factors.append(1) + factors = tuple(factors) + expanded = v.repeat(*factors) + backbone_output[k] = expanded + + for k, v in action_input.items(): + ndim = len(v.shape) + factors = [self.config.expand_batch] + while len(factors) < ndim: + factors.append(1) + factors = tuple(factors) + expanded = v.repeat(*factors) + action_input[k] = expanded + + # Get vision and language embeddings. + vl_embs = backbone_output.backbone_features + device = vl_embs.device + + # Get embodiment ID. + embodiment_id = action_input.embodiment_id + + # Embed state. + state_features = self.state_encoder(action_input.state, embodiment_id) + + # Embed noised action trajectory. + actions = action_input.action + noise = torch.randn(actions.shape, device=actions.device, dtype=actions.dtype) + t = self.sample_time(actions.shape[0], device=actions.device, dtype=actions.dtype) + t = t[:, None, None] # shape (B,1,1) for broadcast + + noisy_trajectory = (1 - t) * noise + t * actions + velocity = actions - noise + + # Convert (continuous) t -> discrete if needed + t_discretized = (t[:, 0, 0] * self.num_timestep_buckets).long() + action_features = self.action_encoder(noisy_trajectory, t_discretized, embodiment_id) + + # Maybe add position embedding. + if self.config.add_pos_embed: + pos_ids = torch.arange(action_features.shape[1], dtype=torch.long, device=device) + pos_embs = self.position_embedding(pos_ids).unsqueeze(0) + action_features = action_features + pos_embs + + # Join vision, language, state and action embedding along sequence dimension. + future_tokens = self.future_tokens.weight.unsqueeze(0).expand(vl_embs.shape[0], -1, -1) + sa_embs = torch.cat((state_features, future_tokens, action_features), dim=1) + + vl_attn_mask = backbone_output.backbone_attention_mask + + model_output = self.model( + hidden_states=sa_embs, + encoder_hidden_states=vl_embs, + encoder_attention_mask=vl_attn_mask, + timestep=t_discretized, + return_all_hidden_states=False, # NOTE (YL): not using flare now + ) + pred = self.action_decoder(model_output, embodiment_id) + pred_actions = pred[:, -actions.shape[1] :] + + # Slice out only the action portion of pred and target. + action_mask = action_input.action_mask + loss = F.mse_loss(pred_actions, velocity, reduction="none") * action_mask + loss = loss.sum() / action_mask.sum() + output_dict = { + "loss": loss, + } + return BatchFeature(data=output_dict) + + @torch.no_grad() + def get_action(self, backbone_output: BatchFeature, action_input: BatchFeature) -> BatchFeature: + backbone_output = self.process_backbone_output(backbone_output) + + # Get vision and language embeddings. + vl_embs = backbone_output.backbone_features + embodiment_id = action_input.embodiment_id + + # Embed state. + state_features = self.state_encoder(action_input.state, embodiment_id) + + # Set initial actions as the sampled noise. + batch_size = vl_embs.shape[0] + device = vl_embs.device + actions = torch.randn( + size=(batch_size, self.config.action_horizon, self.config.action_dim), + dtype=vl_embs.dtype, + device=device, + ) + + num_steps = self.num_inference_timesteps + dt = 1.0 / num_steps + + # Run denoising steps. + for t in range(num_steps): + t_cont = t / float(num_steps) # e.g. goes 0, 1/N, 2/N, ... + t_discretized = int(t_cont * self.num_timestep_buckets) + + # Embed noised action trajectory. + timesteps_tensor = torch.full(size=(batch_size,), fill_value=t_discretized, device=device) + action_features = self.action_encoder(actions, timesteps_tensor, embodiment_id) + # Maybe add position embedding. + if self.config.add_pos_embed: + pos_ids = torch.arange(action_features.shape[1], dtype=torch.long, device=device) + pos_embs = self.position_embedding(pos_ids).unsqueeze(0) + action_features = action_features + pos_embs + + # Join vision, language, state and action embedding along sequence dimension. + future_tokens = self.future_tokens.weight.unsqueeze(0).expand(vl_embs.shape[0], -1, -1) + sa_embs = torch.cat((state_features, future_tokens, action_features), dim=1) + + # Run model forward. + model_output = self.model( + hidden_states=sa_embs, + encoder_hidden_states=vl_embs, + timestep=timesteps_tensor, + ) + pred = self.action_decoder(model_output, embodiment_id) + + pred_velocity = pred[:, -self.action_horizon :] + + # Update actions using euler integration. + actions = actions + dt * pred_velocity + return BatchFeature(data={"action_pred": actions}) + + @property + def device(self): + return next(iter(self.parameters())).device + + @property + def dtype(self): + return next(iter(self.parameters())).dtype diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/configuration_groot.py b/eval/pytorch_ref/policies/gr00t_n15/groot/configuration_groot.py new file mode 100644 index 0000000..9478ede --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/configuration_groot.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python + +# Copyright 2024 NVIDIA Corporation and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field + +from lerobot.configs.policies import PreTrainedConfig +from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature +from lerobot.optim.optimizers import AdamWConfig +from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig +from lerobot.utils.constants import ACTION, OBS_STATE + + +@PreTrainedConfig.register_subclass("gr00t_n15") +@dataclass +class GrootConfig(PreTrainedConfig): + """Configuration for Groot policy wrapper.""" + + # Basic policy settings + n_obs_steps: int = 1 + chunk_size: int = 50 + n_action_steps: int = 50 + + # Dimension settings (must match pretrained GR00T model expectations) + # Maximum state dimension. Shorter states will be zero-padded. + max_state_dim: int = 64 + + # Maximum action dimension. Shorter actions will be zero-padded. + max_action_dim: int = 32 + + # Normalization (start with identity, adjust as needed) + normalization_mapping: dict[str, NormalizationMode] = field( + default_factory=lambda: { + "VISUAL": NormalizationMode.IDENTITY, + "STATE": NormalizationMode.MEAN_STD, + "ACTION": NormalizationMode.MEAN_STD, + } + ) + + # Image preprocessing (adjust to match Groot's expected input) + image_size: tuple[int, int] = (224, 224) + + # Groot-specific model parameters (from groot_finetune_script.py) + + # Path or HuggingFace model ID for the base Groot model + base_model_path: str = "nvidia/GR00T-N1.5-3B" + + # HF repo ID (or local path) that hosts vocab.json and merges.txt for Eagle tokenizer. + tokenizer_assets_repo: str = "lerobot/eagle2hg-processor-groot-n1p5" + + # Embodiment tag to use for training (e.g. 'new_embodiment', 'gr1') + embodiment_tag: str = "new_embodiment" + + # Fine-tuning control arguments + + # Whether to fine-tune the llm backbone + tune_llm: bool = False + + # Whether to fine-tune the vision tower + tune_visual: bool = False + + # Whether to fine-tune the projector + tune_projector: bool = True + + # Whether to fine-tune the diffusion model + tune_diffusion_model: bool = True + + # LoRA parameters (from groot_finetune_script.py) + # Rank for the LORA model. If 0, no LORA will be used. + lora_rank: int = 0 + + # Alpha value for the LORA model + lora_alpha: int = 16 + + # Dropout rate for the LORA model + lora_dropout: float = 0.1 + + # Whether to use the full model for LORA + lora_full_model: bool = False + + # Training parameters (matching groot_finetune_script.py) + optimizer_lr: float = 1e-4 + optimizer_betas: tuple[float, float] = (0.95, 0.999) + optimizer_eps: float = 1e-8 + optimizer_weight_decay: float = 1e-5 + warmup_ratio: float = 0.05 + use_bf16: bool = True + + # Dataset parameters + # Video backend to use for training ('decord' or 'torchvision_av') + video_backend: str = "decord" + + # Whether to balance dataset weights in mixture datasets + balance_dataset_weights: bool = True + + # Whether to sample trajectories weighted by their length + balance_trajectory_weights: bool = True + + # Optional dataset paths for delegating training to Isaac-GR00T runner + dataset_paths: list[str] | None = None + output_dir: str = "./tmp/gr00t" + save_steps: int = 1000 + max_steps: int = 10000 + batch_size: int = 32 + dataloader_num_workers: int = 8 + report_to: str = "wandb" + resume: bool = False + + def __post_init__(self): + super().__post_init__() + + if self.n_action_steps > self.chunk_size: + raise ValueError( + f"n_action_steps ({self.n_action_steps}) cannot exceed chunk_size ({self.chunk_size})" + ) + + # groot_repo_path is now optional since we ported the components + # No validation needed + + def validate_features(self) -> None: + """Validate and set up input/output features for Groot.""" + image_features = [key for key, feat in self.input_features.items() if feat.type == FeatureType.VISUAL] + if not image_features: + raise ValueError( + "Groot policy requires at least one visual input feature. " + "No features of type FeatureType.VISUAL found in input_features." + ) + + if OBS_STATE not in self.input_features: + state_feature = PolicyFeature( + type=FeatureType.STATE, + shape=(self.max_state_dim,), + ) + self.input_features[OBS_STATE] = state_feature + else: + state_shape = self.input_features[OBS_STATE].shape + state_dim = state_shape[0] if state_shape else 0 + if state_dim > self.max_state_dim: + raise ValueError( + f"State dimension {state_dim} exceeds max_state_dim {self.max_state_dim}. " + f"Either reduce state dimension or increase max_state_dim in config." + ) + + if ACTION not in self.output_features: + action_feature = PolicyFeature( + type=FeatureType.ACTION, + shape=(self.max_action_dim,), + ) + self.output_features[ACTION] = action_feature + else: + action_shape = self.output_features[ACTION].shape + action_dim = action_shape[0] if action_shape else 0 + if action_dim > self.max_action_dim: + raise ValueError( + f"Action dimension {action_dim} exceeds max_action_dim {self.max_action_dim}. " + f"Either reduce action dimension or increase max_action_dim in config." + ) + + def get_optimizer_preset(self) -> AdamWConfig: + """Return optimizer configuration.""" + return AdamWConfig( + lr=self.optimizer_lr, + betas=self.optimizer_betas, + eps=self.optimizer_eps, + weight_decay=self.optimizer_weight_decay, + ) + + def get_scheduler_preset(self) -> CosineDecayWithWarmupSchedulerConfig: + """Return scheduler configuration.""" + return CosineDecayWithWarmupSchedulerConfig( + num_warmup_steps=int(10000 * self.warmup_ratio), # 5% warmup by default + num_decay_steps=10000, # Adjust based on training steps + peak_lr=self.optimizer_lr, + decay_lr=self.optimizer_lr * 0.1, + ) + + @property + def observation_delta_indices(self) -> None: + """Return indices for delta observations (None for Groot).""" + return None + + @property + def action_delta_indices(self) -> list[int]: + """Return indices for delta actions.""" + return list(range(min(self.chunk_size, 16))) + + @property + def reward_delta_indices(self) -> None: + """Return indices for delta rewards (None for Groot).""" + return None diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/configuration_eagle2_5_vl.py b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/configuration_eagle2_5_vl.py new file mode 100644 index 0000000..526b4f7 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/configuration_eagle2_5_vl.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy + +from transformers.configuration_utils import PretrainedConfig +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config +from transformers.models.qwen3.configuration_qwen3 import Qwen3Config +from transformers.models.siglip.configuration_siglip import SiglipVisionConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class Eagle25VLConfig(PretrainedConfig): + model_type = "eagle_2_5_vl" + is_composition = True + sub_configs = {"vision_config": SiglipVisionConfig, "text_config": Qwen2Config} + + def __init__( + self, + vision_config=None, + text_config=None, + use_backbone_lora=0, + use_llm_lora=0, + pad2square=False, + select_layer=-4, + force_image_size=None, + downsample_ratio=0.5, + template=None, + dynamic_image_size=False, + use_thumbnail=False, + loss_version="v1", + min_dynamic_tiles=1, + max_dynamic_tiles=6, + mlp_checkpoint=False, + initializer_range=0.02, + _attn_implementation="flash_attention_2", + _attn_implementation_autoset=False, + llm_config=None, + image_token_index=None, + use_pixel_shuffle=True, + mlp_connector_layers=2, + **kwargs, + ): + super().__init__(**kwargs) + + if vision_config is None: + vision_config = {"model_type": "siglip_vision_model"} + logger.info("vision_config is None. Initializing the InternVisionConfig with default values.") + + if text_config is None: + text_config = {"architectures": ["Qwen2ForCausalLM"]} + logger.info( + "text_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`)." + ) + + if vision_config["model_type"] == "siglip_vision_model": + self.vision_config = SiglipVisionConfig(**vision_config) + else: + raise ValueError("Unsupported model_type: {}".format(vision_config["model_type"])) + + if text_config["architectures"][0] == "LlamaForCausalLM": + self.text_config = LlamaConfig(**text_config) + elif text_config["architectures"][0] == "Qwen2ForCausalLM": + self.text_config = Qwen2Config(**text_config) + elif text_config["architectures"][0] == "Qwen3ForCausalLM": + self.text_config = Qwen3Config(**text_config) + else: + raise ValueError("Unsupported architecture: {}".format(text_config["architectures"][0])) + self.use_backbone_lora = use_backbone_lora + self.use_llm_lora = use_llm_lora + self.mlp_checkpoint = mlp_checkpoint + self.pad2square = pad2square + self.select_layer = select_layer + self.force_image_size = force_image_size + self.downsample_ratio = downsample_ratio + self.template = template + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.loss_version = loss_version + self.initializer_range = initializer_range + self.min_dynamic_tiles = min_dynamic_tiles + self.max_dynamic_tiles = max_dynamic_tiles + self.tie_word_embeddings = self.text_config.tie_word_embeddings + self._attn_implementation = _attn_implementation + self._attn_implementation_autoset = _attn_implementation_autoset + self.image_token_index = image_token_index + self.use_pixel_shuffle = use_pixel_shuffle + self.mlp_connector_layers = mlp_connector_layers + logger.info(f"min_dynamic_tiles: {self.min_dynamic_tiles}") + logger.info(f"max_dynamic_tiles: {self.max_dynamic_tiles}") + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["vision_config"] = self.vision_config.to_dict() + output["text_config"] = self.text_config.to_dict() + output["model_type"] = self.__class__.model_type + output["use_backbone_lora"] = self.use_backbone_lora + output["use_llm_lora"] = self.use_llm_lora + output["pad2square"] = self.pad2square + output["select_layer"] = self.select_layer + output["force_image_size"] = self.force_image_size + output["downsample_ratio"] = self.downsample_ratio + output["template"] = self.template + output["dynamic_image_size"] = self.dynamic_image_size + output["use_thumbnail"] = self.use_thumbnail + output["min_dynamic_tiles"] = self.min_dynamic_tiles + output["max_dynamic_tiles"] = self.max_dynamic_tiles + output["tie_word_embeddings"] = self.tie_word_embeddings + output["_attn_implementation"] = self._attn_implementation + output["_attn_implementation_autoset"] = self._attn_implementation_autoset + output["use_pixel_shuffle"] = self.use_pixel_shuffle + output["mlp_connector_layers"] = self.mlp_connector_layers + return output diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/image_processing_eagle2_5_vl_fast.py b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/image_processing_eagle2_5_vl_fast.py new file mode 100644 index 0000000..6bb1f62 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/image_processing_eagle2_5_vl_fast.py @@ -0,0 +1,504 @@ +# -------------------------------------------------------- +# NVIDIA +# Copyright (c) 2025 NVIDIA +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +# copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py +from typing import Optional + +from transformers.image_processing_utils import ( + BatchFeature, + get_patch_output_size, +) +from transformers.image_processing_utils_fast import ( + BaseImageProcessorFast, + DefaultFastImageProcessorKwargs, + group_images_by_shape, + reorder_images, +) +from transformers.image_utils import ( + IMAGENET_STANDARD_MEAN, # 0.5, 0.5, 0.5 + IMAGENET_STANDARD_STD, # 0.5, 0.5, 0.5 + ChannelDimension, + ImageInput, + PILImageResampling, + SizeDict, + get_image_size, + make_flat_list_of_images, + validate_kwargs, +) +from transformers.processing_utils import Unpack +from transformers.utils import ( + TensorType, + add_start_docstrings, + is_torch_available, + is_torchvision_v2_available, +) +from transformers.video_utils import VideoInput + +if is_torch_available(): + import torch +if is_torchvision_v2_available(): + from torchvision.transforms.v2 import functional as F # noqa: N812 + from transformers.image_utils import pil_torch_interpolation_mapping +else: + from torchvision.transforms import functional as F # noqa: N812 + + +def crop(img: torch.Tensor, left: int, top: int, right: int, bottom: int) -> torch.Tensor: + """Crop the given numpy array. + + Args: + img (torch.Tensor): Image to be cropped. Format should be (C, H, W). + left (int): The left coordinate of the crop box. + top (int): The top coordinate of the crop box. + right (int): The right coordinate of the crop box. + bottom (int): The bottom coordinate of the crop box. + + Returns: + torch.Tensor: Cropped image. + """ + if not isinstance(img, torch.Tensor): + raise TypeError(f"img should be torch.Tensor. Got {type(img)}") + + if img.ndim not in [2, 3]: + raise ValueError(f"Image should have 2 or 3 dimensions. Got {img.ndim}") + + img_height = img.shape[1] + img_width = img.shape[2] + if top < 0 or left < 0 or bottom > img_height or right > img_width: + raise ValueError("Crop coordinates out of bounds") + + if top >= bottom or left >= right: + raise ValueError("Invalid crop coordinates") + + return img[:, top:bottom, left:right] + + +class Eagle25VLFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + max_dynamic_tiles: int | None + min_dynamic_tiles: int | None + use_thumbnail: bool | None + pad_during_tiling: bool | None + do_pad: bool | None + + +@add_start_docstrings( + "Constructs a fast ConvNeXT image processor. Based on [`SiglipImageProcessor`] with incorporation of processing each video frame.", + # BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, TODO: this was depreciated from transformers remove! + """ + image_grid_pinpoints (`List[List[int]]`, *optional*): + A list of possible resolutions to use for processing high resolution images. The best resolution is selected + based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` + method. Not used for processing videos. + do_pad (`bool`, *optional*): + Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest + number of patches in the batch. Padding will be applied to the bottom and right with zeros. + """, +) +class Eagle25VLImageProcessorFast(BaseImageProcessorFast): + resample = PILImageResampling.BICUBIC + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + size = {"height": 448, "width": 448} + default_to_square = False + crop_size = None + do_resize = True + do_center_crop = None + do_rescale = True + do_normalize = True + do_convert_rgb = True + do_pad = True + max_dynamic_tiles = 12 + min_dynamic_tiles = 1 + use_thumbnail = True + pad_during_tiling = False + valid_kwargs = Eagle25VLFastImageProcessorKwargs + model_input_names = ["pixel_values_videos"] + + def __init__(self, **kwargs: Unpack[Eagle25VLFastImageProcessorKwargs]): + super().__init__(**kwargs) + + @add_start_docstrings( + # BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, TODO: this was depreciated from transformers remove! + """ + max_dynamic_tiles (`int`, *optional*): + The maximum number of dynamic tiles to use for processing high resolution images. + min_dynamic_tiles (`int`, *optional*): + The minimum number of dynamic tiles to use for processing high resolution images. + use_thumbnail (`bool`, *optional*): + Whether to use a thumbnail for processing high resolution images. + pad_during_tiling (`bool`, *optional*): + Whether to pad the image during tiling. + do_pad (`bool`, *optional*): + Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest + number of patches in the batch. Padding will be applied to the bottom and right with zeros. + """, + ) + + # NOTE(YL): we will overload the preprocess method to add the image_flags + # def preprocess( + # self, images: ImageInput, **kwargs: Unpack[Eagle25VLFastImageProcessorKwargs] + # ) -> BatchFeature: + # return super().preprocess(images, **kwargs) + + def _prepare_images_structure( + self, + images: ImageInput, + expected_ndims: int = 3, + ) -> ImageInput: + """ + Prepare the images structure for processing. + + Args: + images (`ImageInput`): + The input images to process. + expected_ndims (`int`, *optional*, defaults to 3): + Expected number of dimensions for the images (added for transformers >=4.53.0 compatibility). + + Returns: + `ImageInput`: The images with a valid nesting. + """ + return make_flat_list_of_images(images) + + def _resize_for_patching( + self, + image: "torch.Tensor", + target_resolution: tuple, + interpolation: "F.InterpolationMode", + input_data_format: ChannelDimension, + ) -> "torch.Tensor": + """ + Resizes an image to a target resolution while maintaining aspect ratio. + + Args: + image ("torch.Tensor"): + The input image. + target_resolution (tuple): + The target resolution (height, width) of the image. + interpolation (`InterpolationMode`): + Resampling filter to use if resizing the image. + input_data_format (`ChannelDimension` or `str`): + The channel dimension format of the input image. + + Returns: + "torch.Tensor": The resized and padded image. + """ + new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format) + + # Resize the image + resized_image = F.resize(image, (new_height, new_width), interpolation=interpolation) + + return resized_image + + def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size): + """ + previous version mainly focus on ratio. + We also consider area ratio here. + """ + best_factor = float("-inf") + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + # ratio_diff = abs(aspect_ratio - target_aspect_ratio) + # area_ratio = (ratio[0] * ratio[1] * image_size * image_size) / area + """ + new area > 60% of original image area is enough. + """ + factor_based_on_area_n_ratio = min( + (ratio[0] * ratio[1] * image_size * image_size) / area, 0.6 + ) * min(target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio) + + if factor_based_on_area_n_ratio > best_factor: + best_factor = factor_based_on_area_n_ratio + best_ratio = ratio + + return best_ratio + + def _pad_for_patching( + self, image: "torch.Tensor", target_resolution: tuple, input_data_format: ChannelDimension + ) -> "torch.Tensor": + """ + Pad an image to a target resolution while maintaining aspect ratio. + """ + target_height, target_width = target_resolution + new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format) + + paste_x = (target_width - new_width) // 2 + paste_y = (target_height - new_height) // 2 + + padded_image = F.pad(image, padding=[paste_x, paste_y, paste_x, paste_y]) + + return padded_image + + def _get_image_patches( + self, + image: "torch.Tensor", + min_num: int, + max_num: int, + size: tuple, + tile_size: int, + use_thumbnail: bool, + interpolation: "F.InterpolationMode", + pad_during_tiling: bool, + ) -> list["torch.Tensor"]: + image_size = get_image_size(image, channel_dim=ChannelDimension.FIRST) + orig_height, orig_width = image_size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = { + (i, j) + for n in range(min_num, max_num + 1) + for i in range(1, n + 1) + for j in range(1, n + 1) + if i * j <= max_num and i * j >= min_num + } + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = self.find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, tile_size + ) + + # calculate the target width and height + target_width = tile_size * target_aspect_ratio[0] + target_height = tile_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + if pad_during_tiling: + resized_image = self._resize_for_patching( + image, + (target_height, target_width), + interpolation=interpolation, + input_data_format=ChannelDimension.FIRST, + ) + padded_image = self._pad_for_patching( + resized_image, + (target_height, target_width), + input_data_format=ChannelDimension.FIRST, + ) + image_used_to_split = padded_image + else: + image_used_to_split = F.resize(image, (target_height, target_width), interpolation=interpolation) + + processed_tiles = [] + for i in range(blocks): + box = ( + (i % (target_width // tile_size)) * tile_size, + (i // (target_width // tile_size)) * tile_size, + ((i % (target_width // tile_size)) + 1) * tile_size, + ((i // (target_width // tile_size)) + 1) * tile_size, + ) + # split the image + split_img = crop(image_used_to_split, box[0], box[1], box[2], box[3]) + processed_tiles.append(split_img) + assert len(processed_tiles) == blocks + + if use_thumbnail and len(processed_tiles) != 1: + thumbnail_img = F.resize(image, (tile_size, tile_size), interpolation=interpolation) + processed_tiles.append(thumbnail_img) + + return processed_tiles + + def _pad_for_batching( + self, + pixel_values: list["torch.Tensor"], + ) -> list["torch.Tensor"]: + """ + Pads images on the `num_of_patches` dimension with zeros to form a batch of same number of patches. + + Args: + pixel_values (`List[torch.Tensor]`): + An array of pixel values of each images of shape (`batch_size`, `num_patches`, `image_in_3D`) + + Returns: + List[`torch.Tensor`]: The padded images. + """ + max_patch = max(len(x) for x in pixel_values) + pixel_values = [ + torch.nn.functional.pad(image, pad=[0, 0, 0, 0, 0, 0, 0, max_patch - image.shape[0]]) + for image in pixel_values + ] + + return pixel_values + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + max_dynamic_tiles: int, + min_dynamic_tiles: int, + use_thumbnail: bool, + pad_during_tiling: bool, + interpolation: Optional["F.InterpolationMode"], + do_center_crop: bool, + crop_size: SizeDict, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + do_pad: bool, + return_tensors: str | TensorType | None, + pad_size: SizeDict | None = None, # Added for transformers >=4.53.0 compatibility + disable_grouping: bool | None = None, # Added for transformers >=4.53.0 compatibility + ) -> BatchFeature: + processed_images = [] + image_sizes = [] + # Determine the size tuple + if size and size.height and size.width: + size_tuple = (size.height, size.width) + else: + size_tuple = (size.shortest_edge, size.shortest_edge) + + # Determine the patch size + if crop_size and crop_size.height: + tile_size = crop_size.height + elif size and size.height: + tile_size = size.height + else: + tile_size = size.shortest_edge + + for image in images: + image_patches = self._get_image_patches( + image, + min_num=min_dynamic_tiles, + max_num=max_dynamic_tiles, + size=size_tuple, + tile_size=tile_size, + use_thumbnail=use_thumbnail, + interpolation=interpolation, + pad_during_tiling=pad_during_tiling, + ) + + # Group images by size for batched processing + processed_image_patches_grouped = {} + # Added for transformers >=4.53.0 compatibility + grouped_image_patches, grouped_image_patches_index = group_images_by_shape( + image_patches, + disable_grouping=disable_grouping, + ) + + for shape, stacked_image_patches in grouped_image_patches.items(): + if do_resize: + stacked_image_patches = self.resize( + image=stacked_image_patches, + size=size, + interpolation=interpolation, + ) + if do_center_crop: + stacked_image_patches = self.center_crop(stacked_image_patches, crop_size) + # Fused rescale and normalize + stacked_image_patches = self.rescale_and_normalize( + stacked_image_patches, + do_rescale, + rescale_factor, + do_normalize, + image_mean, + image_std, + ) + processed_image_patches_grouped[shape] = stacked_image_patches + processed_image_patches = reorder_images( + processed_image_patches_grouped, grouped_image_patches_index + ) + processed_image_patches = ( + torch.stack(processed_image_patches, dim=0) if return_tensors else processed_image_patches + ) + processed_images.append(processed_image_patches) + image_sizes.append(get_image_size(image, ChannelDimension.FIRST)) + + if do_pad: + processed_images = self._pad_for_batching(processed_images) + + # processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images + processed_images = torch.cat(processed_images, dim=0) if return_tensors else processed_images + return BatchFeature( + data={"pixel_values": processed_images, "image_sizes": image_sizes}, + tensor_type=return_tensors, + ) + + def preprocess( + self, + images: ImageInput, + videos: VideoInput = None, + **kwargs: Unpack[Eagle25VLFastImageProcessorKwargs], + ) -> BatchFeature: + validate_kwargs( + captured_kwargs=kwargs.keys(), + valid_processor_keys=self.valid_kwargs.__annotations__.keys(), + ) + # Set default kwargs from self. This ensures that if a kwarg is not provided + # by the user, it gets its default value from the instance, or is set to None. + for kwarg_name in self.valid_kwargs.__annotations__: + kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None)) + + # Extract parameters that are only used for preparing the input images + do_convert_rgb = kwargs.pop("do_convert_rgb") + input_data_format = kwargs.pop("input_data_format") + device = kwargs.pop("device") + # Prepare input images + # transformers >= 4.53.0: uses _prepare_image_like_inputs instead of _prepare_input_images + if images is not None: + images = self._prepare_input_images( + images=images, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + device=device, + ) + + if videos is not None: + videos = self._prepare_input_images( + images=videos, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + device=device, + ) + + # Update kwargs that need further processing before being validated + kwargs = self._further_process_kwargs(**kwargs) + + # Validate kwargs + self._validate_preprocess_kwargs(**kwargs) + + # torch resize uses interpolation instead of resample + # Added for transformers >=4.53.0 compatibility + resample = kwargs.pop("resample", self.resample) + kwargs["interpolation"] = ( + pil_torch_interpolation_mapping[resample] + if isinstance(resample, PILImageResampling | int) + else resample + ) + + # Filter kwargs to only include those accepted by _preprocess + valid_preprocess_kwargs = { + "do_resize", + "size", + "max_dynamic_tiles", + "min_dynamic_tiles", + "use_thumbnail", + "pad_during_tiling", + "interpolation", + "do_center_crop", + "crop_size", + "do_rescale", + "rescale_factor", + "do_normalize", + "image_mean", + "image_std", + "do_pad", + "return_tensors", + "pad_size", + "disable_grouping", + } + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_preprocess_kwargs} + if images is not None: + return self._preprocess(images, **filtered_kwargs) + elif videos is not None: + return self._preprocess(videos, **filtered_kwargs) + + +__all__ = ["Eagle25VLImageProcessorFast"] diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/modeling_eagle2_5_vl.py b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/modeling_eagle2_5_vl.py new file mode 100644 index 0000000..5a66cfb --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/modeling_eagle2_5_vl.py @@ -0,0 +1,395 @@ +# -------------------------------------------------------- +# NVIDIA +# Copyright (c) 2025 NVIDIA +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import inspect + +import torch +import torch.utils.checkpoint as cp +from peft import LoraConfig, get_peft_model +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers import GenerationConfig +from transformers.generation import GenerationMixin +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.models.llama.modeling_llama import LlamaForCausalLM +from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM +from transformers.models.qwen3.modeling_qwen3 import Qwen3ForCausalLM +from transformers.models.siglip.modeling_siglip import SiglipVisionModel +from transformers.utils import add_start_docstrings, logging + +from .configuration_eagle2_5_vl import Eagle25VLConfig + +logger = logging.get_logger(__name__) + + +# copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/modeling_llava_onevision.py#L241C1-L280C1 +EAGLE2_5_VL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Eagle25VLConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Eagle2_5_VL Model outputting raw hidden-states without any specific head on top.", + EAGLE2_5_VL_START_DOCSTRING, +) +class Eagle25VLPreTrainedModel(PreTrainedModel): + config_class = Eagle25VLConfig + base_model_prefix = "model" + main_input_name = "input_ids" + supports_gradient_checkpointing = True + _no_split_modules = [ + "Qwen2DecoderLayer", + "LlamaDecoderLayer", + "Siglip2EncoderLayer", + "SiglipEncoderLayer", + ] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_cache_class = True + _supports_static_cache = True + _supports_quantized_cache = True + _supports_sdpa = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear | nn.Conv2d): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +class Eagle25VLForConditionalGeneration(Eagle25VLPreTrainedModel, GenerationMixin): + config_class = Eagle25VLConfig + + def __init__(self, config: Eagle25VLConfig, vision_model=None, language_model=None): + super().__init__(config) + + image_size = config.force_image_size or config.vision_config.image_size + patch_size = config.vision_config.patch_size + self.patch_size = patch_size + if config.use_pixel_shuffle: + self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio**2)) + else: + self.num_image_token = int((image_size // patch_size) ** 2) + + self.select_layer = config.select_layer + self.downsample_ratio = config.downsample_ratio + self.loss_version = config.loss_version + self.mlp_checkpoint = config.mlp_checkpoint + self.use_pixel_shuffle = config.use_pixel_shuffle + self.mlp_connector_layers = config.mlp_connector_layers + logger.info(f"num_image_token: {self.num_image_token}") + logger.info(f"mlp_checkpoint: {self.mlp_checkpoint}") + if vision_model is not None: + self.vision_model = vision_model + else: + if config.vision_config.model_type == "siglip_vision_model": + config.vision_config._attn_implementation = "flash_attention_2" + self.vision_model = SiglipVisionModel(config.vision_config) + else: + raise NotImplementedError(f"{config.vision_config.model_type} is not implemented.") + + if language_model is not None: + self.language_model = language_model + else: + if config.text_config.architectures[0] == "LlamaForCausalLM": + self.language_model = LlamaForCausalLM(config.text_config) + elif config.text_config.architectures[0] == "Phi3ForCausalLM": + raise NotImplementedError("Phi3 is not implemented.") + # self.language_model = Phi3ForCausalLM(config.text_config) + elif config.text_config.architectures[0] == "Qwen2ForCausalLM": + assert config.text_config._attn_implementation == "flash_attention_2", ( + f"Qwen2 must use flash_attention_2 but got {config.text_config._attn_implementation}" + ) + self.language_model = Qwen2ForCausalLM(config.text_config) + elif config.text_config.architectures[0] == "Qwen3ForCausalLM": + self.language_model = Qwen3ForCausalLM(config.text_config) + else: + raise NotImplementedError(f"{config.text_config.architectures[0]} is not implemented.") + + vit_hidden_size = config.vision_config.hidden_size + llm_hidden_size = config.text_config.hidden_size + + if config.mlp_connector_layers == 2: + self.mlp1 = nn.Sequential( + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2), + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size), + nn.GELU(), + nn.Linear(llm_hidden_size, llm_hidden_size), + ) + elif config.mlp_connector_layers == 1 and config.use_pixel_shuffle: + self.mlp1 = nn.Sequential( + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size), + ) + elif config.mlp_connector_layers == 1 and not config.use_pixel_shuffle: + self.mlp1 = nn.Sequential( + nn.Linear(vit_hidden_size, llm_hidden_size), + ) + else: + raise NotImplementedError(f"{config.mlp_connector_layers} is not implemented.") + + self.image_token_index = config.image_token_index + self.neftune_alpha = None + + if config.use_backbone_lora: + self.wrap_backbone_lora(r=config.use_backbone_lora, lora_alpha=2 * config.use_backbone_lora) + + self.use_llm_lora = config.use_llm_lora + if config.use_llm_lora: + self.wrap_llm_lora(r=config.use_llm_lora, lora_alpha=2 * config.use_llm_lora) + + self.check_forward_kwargs() + + def check_forward_kwargs(self): + # We intentionally avoid using **kwargs in forward because Hugging Face Transformers + # has special handling for functions with **kwargs parameters that would affect + # how our model is processed during training and inference. + forward_params = inspect.signature(self.forward).parameters + assert not any(k.kind == inspect.Parameter.VAR_KEYWORD for k in forward_params.values()) + + def wrap_backbone_lora(self, r=128, lora_alpha=256, lora_dropout=0.05): + lora_config = LoraConfig( + r=r, + target_modules=[ + "self_attn.q_proj", + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.out_proj", + "mlp.fc1", + "mlp.fc2", + ], + lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + ) + self.vision_model = get_peft_model(self.vision_model, lora_config) + self.vision_model.print_trainable_parameters() + + def wrap_llm_lora(self, r=128, lora_alpha=256, lora_dropout=0.05): + lora_config = LoraConfig( + r=r, + target_modules=[ + "self_attn.q_proj", + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.o_proj", + "mlp.gate_proj", + "mlp.down_proj", + "mlp.up_proj", + ], + lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + task_type="CAUSAL_LM", + ) + self.language_model = get_peft_model(self.language_model, lora_config) + self.language_model.enable_input_require_grads() + self.language_model.print_trainable_parameters() + self.use_llm_lora = True + + def forward( + self, + pixel_values: torch.FloatTensor, + input_ids: torch.LongTensor = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + image_flags: torch.LongTensor | None = None, + past_key_values: list[torch.FloatTensor] | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + num_tiles_list: list[torch.Tensor] | None = None, + ) -> tuple | CausalLMOutputWithPast: + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + input_embeds = self.language_model.get_input_embeddings()(input_ids) + + vit_embeds = self.extract_feature(pixel_values) + + if image_flags is not None: + image_flags = image_flags.view(-1) + vit_embeds = vit_embeds[image_flags == 1] + + b, n, c = input_embeds.shape + input_embeds = input_embeds.reshape(b * n, c) + + input_ids = input_ids.reshape(b * n) + selected = input_ids == self.image_token_index + try: + input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, c) + except Exception as e: + vit_embeds = vit_embeds.reshape(-1, c) + print( + f"warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, " + f"vit_embeds.shape={vit_embeds.shape}" + ) + n_token = selected.sum() + input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds[:n_token] + + input_embeds = input_embeds.reshape(b, n, c) + + outputs = self.language_model( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + logits = outputs.logits + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def pixel_shuffle(self, x, scale_factor=0.5): + n, w, h, c = x.size() + # N, W, H, C --> N, W, H * scale, C // scale + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor)) + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale + x = x.permute(0, 2, 1, 3).contiguous() + # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2) + x = x.view(n, int(h * scale_factor), int(w * scale_factor), int(c / (scale_factor * scale_factor))) + + x = x.permute(0, 2, 1, 3).contiguous() + return x + + def extract_feature(self, pixel_values): + if self.select_layer == -1: + vit_embeds = self.vision_model( + pixel_values=pixel_values, output_hidden_states=False, return_dict=True + ) + if hasattr(vit_embeds, "last_hidden_state"): + vit_embeds = vit_embeds.last_hidden_state + + else: + vit_embeds = self.vision_model( + pixel_values=pixel_values, output_hidden_states=True, return_dict=True + ).hidden_states[self.select_layer] + + if self.use_pixel_shuffle: + h = w = int(vit_embeds.shape[1] ** 0.5) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1) + vit_embeds = self.pixel_shuffle( + vit_embeds, scale_factor=self.downsample_ratio + ) # torch.Size([B, 1024, 1024]) -> torch.Size([B, 16, 16, 4096]) + vit_embeds = vit_embeds.reshape( + vit_embeds.shape[0], -1, vit_embeds.shape[-1] + ) # torch.Size([B, 16, 16, 4096]) -> torch.Size([B, 256, 4096]) + + if self.mlp_checkpoint and vit_embeds.requires_grad: + vit_embeds = cp.checkpoint(self.mlp1, vit_embeds) + else: + vit_embeds = self.mlp1(vit_embeds) + + return vit_embeds + + @torch.no_grad() + def generate( + self, + pixel_values: torch.FloatTensor | None = None, + input_ids: torch.FloatTensor | None = None, + attention_mask: torch.LongTensor | None = None, + visual_features: torch.FloatTensor | None = None, + generation_config: GenerationConfig | None = None, + output_hidden_states: bool | None = None, + image_sizes: list[tuple[int, int]] | None = None, + **generate_kwargs, + ) -> torch.LongTensor: + if pixel_values is not None: + if visual_features is not None: + vit_embeds = visual_features + else: + vit_embeds = self.extract_feature(pixel_values) + + input_embeds = self.language_model.get_input_embeddings()(input_ids) + b, n, c = input_embeds.shape + input_embeds = input_embeds.reshape(b * n, c) + + input_ids = input_ids.reshape(b * n) + selected = input_ids == self.config.image_token_index + assert selected.sum() != 0 + input_embeds[selected] = vit_embeds.reshape(-1, c).to(input_embeds.device) + + input_embeds = input_embeds.reshape(b, n, c) + else: + input_embeds = self.language_model.get_input_embeddings()(input_ids) + + if "use_cache" not in generate_kwargs: + generate_kwargs["use_cache"] = True + + outputs = self.language_model.generate( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + generation_config=generation_config, + output_hidden_states=output_hidden_states, + **generate_kwargs, + ) + + return outputs + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_input_embeddings + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_input_embeddings + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_output_embeddings + def get_output_embeddings(self): + return self.language_model.get_output_embeddings() + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_output_embeddings + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_decoder + def set_decoder(self, decoder): + self.language_model.set_decoder(decoder) + + # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_decoder + def get_decoder(self): + return self.language_model.get_decoder() diff --git a/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/processing_eagle2_5_vl.py b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/processing_eagle2_5_vl.py new file mode 100644 index 0000000..27f9b33 --- /dev/null +++ b/eval/pytorch_ref/policies/gr00t_n15/groot/eagle2_hg_model/processing_eagle2_5_vl.py @@ -0,0 +1,518 @@ +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for Eagle25VL. +copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/processing_llava_onevision.py +""" + +import base64 +import os +import re +from io import BytesIO + +import requests +import torch +from PIL import Image +from transformers.feature_extraction_utils import BatchFeature +from transformers.image_utils import ImageInput +from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput +from transformers.utils import logging +from transformers.video_utils import VideoInput + +logger = logging.get_logger(__name__) + + +FRAME_FACTOR = 2 +FPS = 2.0 +FPS_MIN_FRAMES = 4 +FPS_MAX_FRAMES = 256 + + +def to_rgb(pil_image: Image.Image) -> Image.Image: + if pil_image.mode == "RGBA": + white_background = Image.new("RGB", pil_image.size, (255, 255, 255)) + white_background.paste(pil_image, mask=pil_image.split()[3]) # Use alpha channel as mask + return white_background + else: + return pil_image.convert("RGB") + + +def fetch_image(ele: dict[str, str | Image.Image]) -> Image.Image: + image = ele["image"] if "image" in ele else ele["image_url"] + image_obj = None + if isinstance(image, Image.Image): + image_obj = image + elif image.startswith("http://") or image.startswith("https://"): + response = requests.get(image, stream=True, timeout=10) + image_obj = Image.open(BytesIO(response.content)) + elif image.startswith("file://"): + image_obj = Image.open(image[7:]) + elif image.startswith("data:image"): + if "base64," in image: + _, base64_data = image.split("base64,", 1) + data = base64.b64decode(base64_data) + image_obj = Image.open(BytesIO(data)) + else: + image_obj = Image.open(image) + if image_obj is None: + raise ValueError( + f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}" + ) + image = to_rgb(image_obj) + if "scale_factor" in ele: + scale_factor = ele["scale_factor"] + image = image.resize((image.width * scale_factor, image.height * scale_factor), Image.BILINEAR) + return image + + +class Eagle25VLProcessorKwargs(ProcessingKwargs, total=False): + # see processing_utils.ProcessingKwargs documentation for usage. + _defaults = { + "text_kwargs": { + "padding": False, + }, + "images_kwargs": {}, + "videos_kwargs": {"max_dynamic_tiles": 1}, + } + + +class Eagle25VLProcessor(ProcessorMixin): + r""" + Constructs a Eagle25VL processor which wraps a Eagle25VL video processor, Eagle25VL image processor and a Eagle25VL tokenizer into a single processor. + + [`Eagle25VLProcessor`] offers all the functionalities of [`Eagle25VLVideoProcessor`], [`Eagle25VLImageProcessor`] and [`Eagle25VLTokenizer`]. See the + [`~Eagle25VLVideoProcessor.__call__`], [`~Eagle25VLProcessor.__call__`] and [`~Eagle25VLProcessor.decode`] for more information. + + Args: + image_processor ([`LlavaOnevisionImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`LlamaTokenizerFast`], *optional*): + The tokenizer is a required input. + num_image_tokens (`int`, *optional*): + Number of image tokens for one imagethat will be returned by vision tower. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Should be same as in model's config + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. + image_token (`str`, *optional*, defaults to `""`): + Special token used to denote image location. + video_token (`str`, *optional*, defaults to `"