Skip to content

feat(laguna): native GGUF support for poolside Laguna (S/XS) #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lucaspirola wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(laguna): native GGUF support for poolside Laguna (S/XS) #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion python/freetoken/kernel/csrc/gguf/gguf_kernel.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,7 +545,8 @@ torch::Tensor ggml_moe_a8_vec(
int64_t top_k,
int64_t type,
int64_t row,
int64_t tokens) {
int64_t tokens,
int64_t expert_stride_bytes) {
int col = X.sizes()[1];
const int padded = (col + 512 - 1) / 512 * 512;
const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
Expand All @@ -568,6 +569,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 3:
Expand All @@ -581,6 +583,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 6:
Expand All @@ -594,6 +597,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 7:
Expand All @@ -607,6 +611,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 8:
Expand All @@ -620,6 +625,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 10:
Expand All @@ -633,6 +639,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 11:
Expand All @@ -646,6 +653,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 12:
Expand All @@ -659,6 +667,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 13:
Expand All @@ -672,6 +681,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 14:
Expand All @@ -685,6 +695,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 16:
Expand All @@ -698,6 +709,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 17:
Expand All @@ -711,6 +723,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 18:
Expand All @@ -724,6 +737,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 19:
Expand All @@ -737,6 +751,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 20:
Expand All @@ -750,6 +765,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 21:
Expand All @@ -763,6 +779,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 22:
Expand All @@ -776,6 +793,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 23:
Expand All @@ -789,6 +807,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
case 29:
Expand All @@ -802,6 +821,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
expert_stride_bytes,
stream);
break;
}
Expand Down
Loading