diff --git a/.agents/issues/BACKEND-TENSTORRENT-GDN/ISSUE-LOCAL-01M590MZYN28EA6DEKFFA8GNVY.md b/.agents/issues/BACKEND-TENSTORRENT-GDN/ISSUE-LOCAL-01M590MZYN28EA6DEKFFA8GNVY.md new file mode 100644 index 0000000000..3137445ffe --- /dev/null +++ b/.agents/issues/BACKEND-TENSTORRENT-GDN/ISSUE-LOCAL-01M590MZYN28EA6DEKFFA8GNVY.md @@ -0,0 +1,19 @@ +ID: ISSUE-LOCAL-01M590MZYN28EA6DEKFFA8GNVY +Title: the tt-metal pin advance moved f32 ttnn::sum onto the tf32 FPU tile-reduce — the composed GDN decode step lost its f32-exact dot +Row: BACKEND-TENSTORRENT-GDN +State: OPEN +Kind: bug +GitHub: - +Mirror: PENDING +Availability: FULL +Created: 2026-10-19 +Updated: 2026-10-19 +Closed: - + +## Problem + +After the tt-metal stack advance (252ca04f8, tt_umd 0.9.12 era /tmp/umdtrial-install, built 2026-10-09) the wide-range arm of kGdnDecode fails deterministically on the P150: state max_abs 0.0051, max_rel 1.83 vs tol 0.002/abs 1e-5 (test_tenstorrent_backend.cpp:4126; 3/3 identical runs, isolated and in-suite). GDN product code is unchanged since the pin advance; the composed step's exact-f32 construction is broadcast multiply + ttnn::sum (tenstorrent_gdn.cpp GdnDecodeStepComposed), and on the new pin f32 ttnn::sum lands on the tf32 FPU tile-reduce — the same truncation class 8ab35b966 documented for ttnn::matmul. The error lands in the small-magnitude delta-rule terms, the exact signature that commit pins the arm for. The VT_TT_GDN_DECODE=chunked diagnostic degrades further (max_abs 0.246), ruling out a composed-only defect. + +## Resolution + +FIXED in this repair: the dot reductions route through ttnn::mean(dim, keep_dim) - Dk instead of ttnn::sum. The pin's mean documents its default as the accurate full-fp32 SFPU path (generic_reductions.hpp), so the reduction is exact again: state rel_rms 3.6e-5 -> 3.2e-8, max_rel 1.83 -> 9.1e-4, focused arm green 2026-10-19. No assertion re-pinned. diff --git a/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M590NESYZ6D3QNF3F34DDXJG.md b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M590NESYZ6D3QNF3F34DDXJG.md new file mode 100644 index 0000000000..1e2dc2e46f --- /dev/null +++ b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M590NESYZ6D3QNF3F34DDXJG.md @@ -0,0 +1,19 @@ +ID: ISSUE-LOCAL-01M590NESYZ6D3QNF3F34DDXJG +Title: the tt-metal pin advance moved f32 ttnn::sum onto the tf32 FPU tile-reduce — the grouped keep-quant exact-f32 decode dot drifts past its envelope +Row: BACKEND-TENSTORRENT-KEEPQUANT +State: OPEN +Kind: bug +GitHub: - +Mirror: PENDING +Availability: FULL +Created: 2026-10-19 +Updated: 2026-10-19 +Closed: - + +## Problem + +After the tt-metal stack advance (252ca04f8, tt_umd 0.9.12 era /tmp/umdtrial-install, built 2026-10-09) the P=1 grouped decode envelope test fails deterministically on the P150: Q4_K N=1024 K=5120 bad=154/1024, worst_rel 0.1307 (test_tenstorrent_backend.cpp:9696; 3/3 identical runs). KEEPQUANT product code is unchanged since the pin advance. The decode (P==1) arm computes the dot as broadcast multiply + f32 ttnn::sum (tenstorrent_keepquant.cpp, both the fused and the chunk tail), and the drift is BIT-IDENTICAL across the fused arm, VT_TT_KEEPQUANT_FUSED=0, and VT_TT_KEEPQUANT_MM_CHAIN=1 — the shared ttnn::sum is the defective stage, not a matmul arm. On the new pin f32 ttnn::sum lands on the tf32 FPU tile-reduce (the truncation class 8ab35b966 documented for ttnn::matmul), so the exact-f32 decode claim no longer holds. + +## Resolution + +FIXED in this repair: both exact-dot sites route through ttnn::mean(dim, keep_dim) - K. The pin's mean documents its default as the accurate full-fp32 SFPU path (generic_reductions.hpp). Q4_K decode worst_rel 0.1307 -> 1.7e-4, bad 154/1024 -> 0/1024, focused arm green 2026-10-19. No assertion re-pinned. diff --git a/src/vt/tenstorrent/tenstorrent_gdn.cpp b/src/vt/tenstorrent/tenstorrent_gdn.cpp index d756315277..e5e079d1de 100644 --- a/src/vt/tenstorrent/tenstorrent_gdn.cpp +++ b/src/vt/tenstorrent/tenstorrent_gdn.cpp @@ -635,19 +635,32 @@ DecodeStepResult GdnDecodeStepComposed(const ttnn::Tensor& S, const ttnn::Tensor // 4e-5). The truncated dot error lands in the small-magnitude elements of // the delta-rule correction (v - S@k) and compounds across the GDN layers — // pinned by the wide-range arm in tests/vt/test_tenstorrent_backend.cpp. - // The eltwise multiply is exact f32 SFPU and ttnn::sum accumulates f32, so - // the three GEMMs route through broadcast multiply + sum (the outer product - // has no reduction at all — one exact product per element): - ttnn::Tensor dot = ttnn::sum(ttnn::multiply(Sd, krow), ttsl::SmallVector{2}, - /*keep_dim=*/true); + // The eltwise multiply is exact f32 SFPU, so the three GEMMs route through + // broadcast multiply + reduce (the outer product has no reduction at all — + // one exact product per element). The reduction is ttnn::mean · Dk, NOT + // ttnn::sum: on the tt-metal pin this backend builds against + // (tt_umd 0.9.12 era, /tmp/umdtrial-install), f32 ttnn::sum lands on the + // tf32 FPU tile-reduce (the header documents the split: mean's default + // fast_and_approximate_mode=false is "the accurate SFPU path (full fp32)", + // true "the faster tf32 FPU path", and sum has no SFPU arm), which + // truncates the products exactly like the matmul MACs the wide-range arm + // exists to exclude. mean over the K dim divides by an exact f32 scalar and + // the rescale multiplies it back — two ulp of rounding against tf32's + // 2^-11 truncation. + const float dkf = static_cast(dk); + ttnn::Tensor dot = ttnn::multiply( + ttnn::mean(ttnn::multiply(Sd, krow), ttsl::SmallVector{2}, + /*keep_dim=*/true), + dkf); ttnn::Tensor vcol = ttnn::reshape(dev_v, ttnn::Shape({bh, dv, 1})); ttnn::Tensor beta = ttnn::reshape(dev_b, ttnn::Shape({bh, 1, 1})); ttnn::Tensor vp = ttnn::multiply(ttnn::subtract(vcol, dot), beta); ttnn::Tensor S2 = ttnn::add(Sd, ttnn::multiply(vp, krow)); ttnn::Tensor qs = ttnn::multiply(dev_q, scale); - ttnn::Tensor o = ttnn::sum( - ttnn::multiply(S2, ttnn::reshape(qs, ttnn::Shape({bh, 1, dk}))), - ttsl::SmallVector{2}, /*keep_dim=*/true); + ttnn::Tensor o = ttnn::multiply( + ttnn::mean(ttnn::multiply(S2, ttnn::reshape(qs, ttnn::Shape({bh, 1, dk}))), + ttsl::SmallVector{2}, /*keep_dim=*/true), + dkf); return {std::move(o), std::move(S2)}; } diff --git a/src/vt/tenstorrent/tenstorrent_keepquant.cpp b/src/vt/tenstorrent/tenstorrent_keepquant.cpp index b049956e27..f78491bda6 100644 --- a/src/vt/tenstorrent/tenstorrent_keepquant.cpp +++ b/src/vt/tenstorrent/tenstorrent_keepquant.cpp @@ -1412,8 +1412,15 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, ttnn::Tensor wft = ttnn::to_layout(wf, ttnn::Layout::TILE); TTReclaimPlanes(device, {&wf}); ttnn::Tensor prod = ttnn::multiply(wft, dev_a_f32exact); + // mean · K, not ttnn::sum: on the tt-metal pin this backend builds + // against, f32 sum lands on the tf32 FPU tile-reduce while mean's + // default selects the accurate full-fp32 SFPU path (see the header + // comment on generic_reductions.hpp mean and the GDN composed-step + // note in tenstorrent_gdn.cpp). Same reduction, exact accumulation. part = ttnn::permute( - ttnn::sum(prod, ttsl::SmallVector{1}, /*keep_dim=*/true), + ttnn::multiply(ttnn::mean(prod, ttsl::SmallVector{1}, + /*keep_dim=*/true), + static_cast(K)), ttsl::SmallVector{1, 0}); // [rows,1] -> [1,rows] TTReclaimPlanes(device, {&wft, &prod}); } else { @@ -1480,8 +1487,11 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, // bounded), so a forced double free of the shared storage cannot // happen. prod = ttnn::multiply(wft, dev_a_f32exact); + // mean · K, not ttnn::sum — same pin rationale as the fused arm above. part = ttnn::permute( - ttnn::sum(prod, ttsl::SmallVector{1}, /*keep_dim=*/true), + ttnn::multiply(ttnn::mean(prod, ttsl::SmallVector{1}, + /*keep_dim=*/true), + static_cast(K)), ttsl::SmallVector{1, 0}); // [rows,1] -> [1,rows] } else { ttnn::Tensor wbf = ttnn::typecast(wf, ttnn::DataType::BFLOAT16);