perf: add low-latency nz quantized decode matmul.

xllm_ops/build_aclnn.sh

@@ -150,6 +150,7 @@ elif [[ "$SOC_VERSION" =~ ^(ascend)?910b ]]; then
         "moe_init_routing_v3"
         "multi_latent_attention"
         "pp_matmul_opt"
+        "quant_matmul_nz_decode"
         "recurrent_gated_delta_rule"
         "replace_token"
         "mtp_prepare_next_draft"
@@ -266,6 +267,7 @@ elif [[ "$SOC_VERSION" =~ ^ascend910_93 ]]; then
         "moe_grouped_matmul_swiglu_quant"
         "multi_latent_attention"
         "pp_matmul_opt"
+        "quant_matmul_nz_decode"
         "recurrent_gated_delta_rule"
         "replace_token"
         "mtp_prepare_next_draft"
@@ -397,10 +399,39 @@ dump_selected_ops
   : "${SOC_VERSION:?SOC_VERSION is not set}"
   : "${SOC_ARG:?SOC_ARG is not set}"
 
+  if [[ -n "${BUILD_DIR_ARG}" &&
+        ";${CUSTOM_OPS};" == *";quant_matmul_nz_decode;"* ]]; then
+    # The generated kernel targets do not depend on op_kernel sources. Force
+    # OPC to refresh this operator when reusing an incremental build tree.
+    quant_matmul_binary_dir="${BUILD_DIR_ARG}/binary/${SOC_ARG}"
+    rm -f -- \
+      "${quant_matmul_binary_dir}/src/quant_matmul_nz_decode/quant_matmul_nz_decode_${SOC_ARG}_src_copy.done" \
+      "${quant_matmul_binary_dir}/gen/quant_matmul_nz_decode_${SOC_ARG}_0.done" \
+      "${quant_matmul_binary_dir}/gen/quant_matmul_nz_decode_${SOC_ARG}_1.done" \
+      "${quant_matmul_binary_dir}/gen/quant_matmul_nz_decode_${SOC_ARG}_2.done" \
+      "${quant_matmul_binary_dir}/gen/quant_matmul_nz_decode_${SOC_ARG}_3.done"
+
+    quant_matmul_gen_dir="${quant_matmul_binary_dir}/gen"
+    if [[ -d "${quant_matmul_gen_dir}" ]]; then
+      while IFS= read -r -d '' kernel_meta_dir; do
+        find "${kernel_meta_dir}" -depth -delete
+      done < <(find "${quant_matmul_gen_dir}" -mindepth 1 -maxdepth 1 \
+        -type d -name 'kernel_meta_QuantMatmulNzDecode_*' -print0)
+    fi
+
+    quant_matmul_output_dir="${quant_matmul_binary_dir}/bin/quant_matmul_nz_decode"
+    if [[ -d "${quant_matmul_output_dir}" ]]; then
+      find "${quant_matmul_output_dir}" -mindepth 1 -maxdepth 1 \
+        -type f \( -name 'QuantMatmulNzDecode_*.o' \
+                    -o -name 'QuantMatmulNzDecode_*.json' \) -delete
+    fi
+  fi
+
   log "build command: bash build.sh --pkg --ops=\"${CUSTOM_OPS}\" --soc=\"${SOC_ARG}\""
   log "building custom ops ${CUSTOM_OPS} for ${SOC_VERSION}"
   if [ -n "$BUILD_DIR_ARG" ]; then
-    bash build.sh --pkg --ops="${CUSTOM_OPS}" --soc="${SOC_ARG}" --build-dir="${BUILD_DIR_ARG}"
+    bash build.sh --pkg --ops="${CUSTOM_OPS}" --soc="${SOC_ARG}" \
+      --build-dir="${BUILD_DIR_ARG}"
   else
     bash build.sh --pkg --ops="${CUSTOM_OPS}" --soc="${SOC_ARG}"
   fi

xllm_ops/quant_matmul_nz_decode/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Copyright 2026 The xLLM Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+file(GLOB CURRENT_DIRS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
+foreach(SUB_DIR ${CURRENT_DIRS})
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SUB_DIR}/CMakeLists.txt")
+        add_subdirectory(${SUB_DIR})
+    endif()
+endforeach()

xllm_ops/quant_matmul_nz_decode/op_host/CMakeLists.txt

@@ -0,0 +1,39 @@
+# Copyright 2026 The xLLM Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_op_to_compiled_list()
+
+if(BUILD_OPEN_PROJECT)
+    target_sources(op_host_aclnn PRIVATE
+        quant_matmul_nz_decode_def.cpp
+    )
+endif()
+
+add_ops_tiling_keys(
+    OP_NAME QuantMatmulNzDecode
+    TILING_KEYS 1 3 2 0 4 5 6 7 8 9 10 11 12 13 14 15
+)
+
+add_ops_compile_options(
+    OP_NAME QuantMatmulNzDecode
+    OPTIONS --cce-auto-sync=off
+            -DCATLASS_ENABLE_L1_L0_MN_TILE_MISMATCH
+            -Wno-deprecated-declarations
+            -Wno-error
+            -I${CMAKE_SOURCE_DIR}/../third_party/catlass/include
+)
+
+if(NOT BUILD_OPS_RTY_KERNEL)
+    add_modules_sources(OPTYPE quant_matmul_nz_decode ACLNNTYPE aclnn)
+endif()

xllm_ops/quant_matmul_nz_decode/op_host/quant_matmul_nz_decode_def.cpp

@@ -0,0 +1,53 @@
+/* Copyright 2026 The xLLM Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "register/op_def_registry.h"
+
+namespace ops {
+class QuantMatmulNzDecode : public OpDef {
+ public:
+  explicit QuantMatmulNzDecode(const char* name) : OpDef(name) {
+    this->Input("x")
+        .ParamType(REQUIRED)
+        .DataType({ge::DT_INT8})
+        .Format({ge::FORMAT_ND})
+        .UnknownShapeFormat({ge::FORMAT_ND});
+    this->Input("weight")
+        .ParamType(REQUIRED)
+        .DataType({ge::DT_INT8})
+        .Format({ge::FORMAT_FRACTAL_NZ})
+        .UnknownShapeFormat({ge::FORMAT_FRACTAL_NZ});
+    this->Input("scale")
+        .ParamType(REQUIRED)
+        .DataType({ge::DT_FLOAT})
+        .Format({ge::FORMAT_ND})
+        .UnknownShapeFormat({ge::FORMAT_ND});
+    this->Input("bias")
+        .ParamType(REQUIRED)
+        .DataType({ge::DT_INT32})
+        .Format({ge::FORMAT_ND})
+        .UnknownShapeFormat({ge::FORMAT_ND});
+    this->Output("y")
+        .ParamType(REQUIRED)
+        .DataType({ge::DT_BF16})
+        .Format({ge::FORMAT_ND})
+        .UnknownShapeFormat({ge::FORMAT_ND});
+    this->AICore().AddConfig("ascend910b");
+    this->AICore().AddConfig("ascend910_93");
+  }
+};
+
+OP_ADD(QuantMatmulNzDecode);
+}  // namespace ops

xllm_ops/quant_matmul_nz_decode/op_host/quant_matmul_nz_decode_proto.cpp

@@ -0,0 +1,38 @@
+/* Copyright 2026 The xLLM Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "quant_matmul_nz_decode_tiling.h"
+#include "register/op_def_registry.h"
+
+namespace ge {
+static graphStatus InferShape(gert::InferShapeContext* context) {
+  const gert::Shape* x_shape = context->GetInputShape(0);
+  const gert::Shape* weight_shape = context->GetInputShape(1);
+  gert::Shape* y_shape = context->GetOutputShape(0);
+  *y_shape = *x_shape;
+  y_shape->SetDim(y_shape->GetDimNum() - 1,
+                  weight_shape->GetDim(weight_shape->GetDimNum() - 1));
+  return GRAPH_SUCCESS;
+}
+
+static graphStatus InferDataType(gert::InferDataTypeContext* context) {
+  context->SetOutputDataType(0, ge::DT_BF16);
+  return GRAPH_SUCCESS;
+}
+
+IMPL_OP(QuantMatmulNzDecode)
+    .InferShape(InferShape)
+    .InferDataType(InferDataType);
+}  // namespace ge

xllm_ops/quant_matmul_nz_decode/op_host/quant_matmul_nz_decode_tiling.cpp

@@ -0,0 +1,122 @@
+/* Copyright 2026 The xLLM Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "quant_matmul_nz_decode_tiling.h"
+
+#include "register/op_def_registry.h"
+#include "tiling/platform/platform_ascendc.h"
+
+namespace optiling {
+namespace {
+constexpr uint32_t kCoreCount = 20;
+constexpr uint32_t kGateUpCoreCount = 20;
+constexpr uint32_t kQkvCoreCount = 10;
+constexpr uint32_t kTileM = 16;
+constexpr uint32_t kMaxOptimizedM = 16;
+constexpr uint32_t kTileN = 320;
+constexpr uint32_t kWorkspaceStages = 2;
+constexpr uint64_t kGateUpM1TilingKey = 6;
+constexpr uint64_t kGateUpM2TilingKey = 7;
+constexpr uint64_t kGateUpM4TilingKey = 8;
+constexpr uint64_t kGateUpM8TilingKey = 9;
+constexpr uint64_t kGateUpM16TilingKey = 10;
+constexpr uint64_t kQkvM1TilingKey = 11;
+constexpr uint64_t kQkvM2TilingKey = 12;
+constexpr uint64_t kQkvM4TilingKey = 13;
+constexpr uint64_t kQkvM8TilingKey = 14;
+constexpr uint64_t kQkvM16TilingKey = 15;
+
+ge::graphStatus TilingFunc(gert::TilingContext* context) {
+  const auto x_shape = context->GetInputShape(0)->GetOriginShape();
+  const auto weight_shape = context->GetInputShape(1)->GetOriginShape();
+  const auto scale_shape = context->GetInputShape(2)->GetOriginShape();
+  const auto bias_shape = context->GetInputShape(3)->GetOriginShape();
+  const size_t x_dim = x_shape.GetDimNum();
+  const size_t weight_dim = weight_shape.GetDimNum();
+  if (x_dim != 2 || weight_dim != 2 || scale_shape.GetDimNum() != 1 ||
+      bias_shape.GetDimNum() != 1) {
+    return ge::GRAPH_FAILED;
+  }
+
+  const int64_t m_dim = x_shape.GetDim(0);
+  const int64_t k_dim = x_shape.GetDim(1);
+  const int64_t n_dim = weight_shape.GetDim(1);
+
+  const bool is_gate_up_shape = k_dim == 5120 && n_dim == 6400;
+  const bool is_down_shape = k_dim == 3200 && n_dim == 5120;
+  const bool is_qkv_shape = k_dim == 5120 && n_dim == 1280;
+  if (m_dim <= 0 || m_dim > kMaxOptimizedM ||
+      (!is_gate_up_shape && !is_down_shape && !is_qkv_shape) ||
+      weight_shape.GetDim(0) != k_dim || scale_shape.GetDim(0) != n_dim ||
+      bias_shape.GetDim(0) != n_dim) {
+    return ge::GRAPH_FAILED;
+  }
+
+  const uint32_t m = static_cast<uint32_t>(m_dim);
+  const uint32_t k = static_cast<uint32_t>(k_dim);
+  const uint32_t n = static_cast<uint32_t>(n_dim);
+  QuantMatmulNzDecodeTilingData tiling;
+  tiling.set_m(m);
+  tiling.set_k(k);
+  tiling.set_n(n);
+  tiling.SaveToBuffer(context->GetRawTilingData()->GetData(),
+                      context->GetRawTilingData()->GetCapacity());
+  context->GetRawTilingData()->SetDataSize(tiling.GetDataSize());
+  context->SetBlockDim(
+      is_qkv_shape ? kQkvCoreCount
+                   : (is_gate_up_shape ? kGateUpCoreCount : kCoreCount));
+  if (is_down_shape) {
+    context->SetTilingKey(m == 1 ? 1 : (m <= 4 ? 3 : 2));
+  } else if (is_qkv_shape) {
+    if (m == 1) {
+      context->SetTilingKey(kQkvM1TilingKey);
+    } else if (m == 2) {
+      context->SetTilingKey(kQkvM2TilingKey);
+    } else if (m == 4) {
+      context->SetTilingKey(kQkvM4TilingKey);
+    } else if (m == 8) {
+      context->SetTilingKey(kQkvM8TilingKey);
+    } else if (m == 16) {
+      context->SetTilingKey(kQkvM16TilingKey);
+    } else {
+      context->SetTilingKey(4);
+    }
+  } else if (m == 1) {
+    context->SetTilingKey(kGateUpM1TilingKey);
+  } else if (m == 2) {
+    context->SetTilingKey(kGateUpM2TilingKey);
+  } else if (m == 4) {
+    context->SetTilingKey(kGateUpM4TilingKey);
+  } else if (m == 8) {
+    context->SetTilingKey(kGateUpM8TilingKey);
+  } else if (m == 16) {
+    context->SetTilingKey(kGateUpM16TilingKey);
+  } else if (m > 2) {
+    context->SetTilingKey(5);
+  } else {
+    context->SetTilingKey(0);
+  }
+
+  auto platform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo());
+  size_t* workspace_size = context->GetWorkspaceSizes(1);
+  workspace_size[0] =
+      platform.GetLibApiWorkSpaceSize() +
+      kTileM * kTileN * kGateUpCoreCount * kWorkspaceStages * sizeof(int32_t);
+  return ge::GRAPH_SUCCESS;
+}
+}  // namespace
+
+IMPL_OP_OPTILING(QuantMatmulNzDecode).Tiling(TilingFunc);
+}  // namespace optiling

xllm_ops/quant_matmul_nz_decode/op_host/quant_matmul_nz_decode_tiling.h

@@ -0,0 +1,29 @@
+/* Copyright 2026 The xLLM Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://gitcode.com/xLLM-AI/xllm_ops/blob/main/LICENSE
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#pragma once
+
+#include "register/tilingdata_base.h"
+
+namespace optiling {
+BEGIN_TILING_DATA_DEF(QuantMatmulNzDecodeTilingData)
+  TILING_DATA_FIELD_DEF(uint32_t, m);
+  TILING_DATA_FIELD_DEF(uint32_t, k);
+  TILING_DATA_FIELD_DEF(uint32_t, n);
+END_TILING_DATA_DEF;
+
+REGISTER_TILING_DATA_CLASS(QuantMatmulNzDecode,
+                           QuantMatmulNzDecodeTilingData)
+}  // namespace optiling