Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 96 additions & 3 deletions common/sampling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ struct common_sampler {
// for rejection sampling; independent of the draft, or the target distribution is not preserved
std::mt19937 rng;

bool pure_argmax_fast = false;

void reset() {
prev.clear();

Expand Down Expand Up @@ -342,10 +344,56 @@ struct common_sampler * common_sampler_init(
}
}

bool pure_argmax_fast = false;

if (params.mirostat == 0) {

bool use_adaptive_p = false; // see below

const bool greedy = params.n_probs == 0 && !params.samplers.empty() &&
((params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && params.temp <= 0.0f && params.dynatemp_range == 0.0f) ||
(params.samplers.back() == COMMON_SAMPLER_TYPE_TOP_K && params.top_k == 1));

bool pure_argmax_chain = greedy && !grmr && !rbudget && !getenv("GGML_VK_DISABLE_GREEDY_FAST");
if (pure_argmax_chain) {
for (const auto & cnstr : params.samplers) {
switch (cnstr) {
case COMMON_SAMPLER_TYPE_TOP_K:
break;
case COMMON_SAMPLER_TYPE_TOP_P:
case COMMON_SAMPLER_TYPE_MIN_P:
if (params.min_keep < 1) { pure_argmax_chain = false; }
break;
case COMMON_SAMPLER_TYPE_TYPICAL_P:
if (params.typ_p < 1.0f) { pure_argmax_chain = false; }
break;
case COMMON_SAMPLER_TYPE_TOP_N_SIGMA:
if (params.top_n_sigma > 0.0f) { pure_argmax_chain = false; }
break;
case COMMON_SAMPLER_TYPE_TEMPERATURE:
if (params.temp > 0.0f || params.dynatemp_range != 0.0f) { pure_argmax_chain = false; }
break;
case COMMON_SAMPLER_TYPE_PENALTIES:
if (params.penalty_last_n != 0 &&
(params.penalty_repeat != 1.0f || params.penalty_freq != 0.0f || params.penalty_present != 0.0f)) {
pure_argmax_chain = false;
}
break;
case COMMON_SAMPLER_TYPE_DRY:
if (params.dry_multiplier > 0.0f) { pure_argmax_chain = false; }
break;
case COMMON_SAMPLER_TYPE_XTC:
if (params.xtc_probability > 0.0f) { pure_argmax_chain = false; }
break;
default:
pure_argmax_chain = false;
break;
}
}
}
pure_argmax_fast = pure_argmax_chain && samplers.empty();

if (!pure_argmax_chain) {
for (const auto & cnstr : params.samplers) {
switch (cnstr) {
case COMMON_SAMPLER_TYPE_DRY:
Expand Down Expand Up @@ -396,14 +444,12 @@ struct common_sampler * common_sampler_init(
GGML_ASSERT(false && "unknown sampler type");
}
}
}
if (use_adaptive_p) {
// only if user explicitly included adaptive-p sampler
samplers.push_back(llama_sampler_init_adaptive_p(params.adaptive_target, params.adaptive_decay, params.seed));
} else {
// Keep distribution sampling when callers request probabilities.
const bool greedy = params.n_probs == 0 && !params.samplers.empty() &&
((params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && params.temp == 0.0f && params.dynatemp_range == 0.0f) ||
(params.samplers.back() == COMMON_SAMPLER_TYPE_TOP_K && params.top_k == 1));
samplers.push_back(greedy ? llama_sampler_init_greedy() : llama_sampler_init_dist(params.seed));
}
} else if (params.mirostat == 1) {
Expand Down Expand Up @@ -442,6 +488,7 @@ struct common_sampler * common_sampler_init(
/* .cur_p = */ {},
// mix it, the chain and the draft are seeded from this one too
/* .rng = */ std::mt19937(llama_sampler_get_seed(chain) ^ 0x9e3779b9u),
/* .pure_argmax_fast = */ pure_argmax_fast,
};

return result;
Expand Down Expand Up @@ -526,6 +573,7 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
/* .cur = */ gsmpl->cur,
/* .cur_p = */ gsmpl->cur_p,
/* .rng = */ gsmpl->rng,
/* .pure_argmax_fast = */ gsmpl->pure_argmax_fast,
};
}

Expand All @@ -547,6 +595,7 @@ void common_sampler_copy(const common_sampler * src, common_sampler * dst) {
dst->cur_p = src->cur_p;
dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer
dst->rng = src->rng;
dst->pure_argmax_fast = src->pure_argmax_fast;
dst->t_total_us = src->t_total_us;
}

Expand Down Expand Up @@ -617,6 +666,28 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co
auto & chain = gsmpl->chain;
auto & cur_p = gsmpl->cur_p; // initialized by set_logits

if (gsmpl->pure_argmax_fast && !gsmpl->grmr && !gsmpl->rbudget &&
llama_get_sampled_token_ith(ctx, idx) == LLAMA_TOKEN_NULL &&
!llama_get_sampled_probs_ith(ctx, idx) &&
!llama_get_sampled_logits_ith(ctx, idx)) {
const float * logits = llama_get_logits_ith(ctx, idx);
GGML_ASSERT(logits != nullptr);
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx)));
llama_token best_id = 0;
float best_val = logits[0];
for (llama_token token_id = 1; token_id < n_vocab; ++token_id) {
const float v = logits[token_id];
if (v > best_val) {
best_val = v;
best_id = token_id;
}
}
gsmpl->cur.resize(1);
gsmpl->cur[0] = llama_token_data{ best_id, best_val, 1.0f };
cur_p = { gsmpl->cur.data(), 1, 0, true };
return best_id;
}

gsmpl->set_logits(ctx, idx);

// Check if a backend sampler has already sampled a token in which case we
Expand Down Expand Up @@ -671,6 +742,28 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co

// resampling:
// if the token is not valid, sample again, but first apply the grammar sampler and then the sampling chain
if (gsmpl->pure_argmax_fast && !gsmpl->grmr && !gsmpl->rbudget &&
llama_get_sampled_token_ith(ctx, idx) == LLAMA_TOKEN_NULL &&
!llama_get_sampled_probs_ith(ctx, idx) &&
!llama_get_sampled_logits_ith(ctx, idx)) {
const float * logits = llama_get_logits_ith(ctx, idx);
GGML_ASSERT(logits != nullptr);
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx)));
llama_token best_id = 0;
float best_val = logits[0];
for (llama_token token_id = 1; token_id < n_vocab; ++token_id) {
const float v = logits[token_id];
if (v > best_val) {
best_val = v;
best_id = token_id;
}
}
gsmpl->cur.resize(1);
gsmpl->cur[0] = llama_token_data{ best_id, best_val, 1.0f };
cur_p = { gsmpl->cur.data(), 1, 0, true };
return best_id;
}

gsmpl->set_logits(ctx, idx);

llama_sampler_apply(rbudget, &cur_p);
Expand Down
6 changes: 5 additions & 1 deletion common/speculative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ static void spec_retune(
sparams.temp = cur.temp;
// must be explicit, the default reseeds at random; mixed so it differs from the target's
sparams.seed = cur.seed == LLAMA_DEFAULT_SEED ? cur.seed : cur.seed ^ 0x85ebca6bu;
sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K, COMMON_SAMPLER_TYPE_TEMPERATURE };
if (cur.temp <= 0.0f && !getenv("GGML_VK_DISABLE_GREEDY_FAST")) {
sparams.samplers = { COMMON_SAMPLER_TYPE_TEMPERATURE };
} else {
sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K, COMMON_SAMPLER_TYPE_TEMPERATURE };
}

smpls[seq_id].reset(common_sampler_init(model, sparams));
}
Expand Down
10 changes: 9 additions & 1 deletion ggml/src/ggml-vulkan/ggml-vulkan-common.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, cons
void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst, const ggml_tensor * gate_scale_in = nullptr);
void ggml_vk_top_k_mask(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);
bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx);
bool ggml_vk_use_mul_mat_vec_id(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);

// flash-attn
Expand Down Expand Up @@ -303,6 +303,14 @@ inline void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context&
GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size());
GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants));

ctx->stat_dispatches++;
{
// gen: GGML_VK_GRAPH_STATS=2 lists every dispatch (pipeline and workgroups)
static const int gen_stats_level = getenv("GGML_VK_GRAPH_STATS") ? atoi(getenv("GGML_VK_GRAPH_STATS")) : 0;
if (gen_stats_level >= 2) {
fprintf(stderr, "ggml_vk_dispatch: %s %u %u %u\n", pipeline->name.c_str(), wg0, wg1, wg2);
}
}
const uint32_t descriptor_set_idx = ctx->descriptor_set_idx++;
vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[descriptor_set_idx];

Expand Down
72 changes: 69 additions & 3 deletions ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,20 @@ struct vk_mat_mat_push_constants {
uint32_t padded_N;
};

struct vk_mat_mat_fused_push_constants {
uint32_t M; uint32_t N; uint32_t K;
uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
uint32_t base_work_group_z; uint32_t num_batches;
uint32_t k_split;
uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
uint32_t padded_N;
uint32_t fusion_flags;
uint32_t pair_m0;
uint32_t stride_d2;
uint32_t batch_stride_d2;
};

struct vk_mat_vec_push_constants {
uint32_t ncols;
uint32_t stride_a;
Expand All @@ -29,6 +43,36 @@ struct vk_mat_vec_push_constants {
uint32_t ne12;
uint32_t broadcast2;
uint32_t broadcast3;
uint32_t pair_m0;
uint32_t stride_d2;
uint32_t batch_stride_d2;
uint32_t lora_r;
float lora_scale0;
float lora_scale1;
};

struct vk_mat_mv_mma_push_constants {
uint32_t M;
uint32_t N;
uint32_t K;
uint32_t stride_a;
uint32_t stride_b;
uint32_t stride_d;
uint32_t batch_stride_a;
uint32_t batch_stride_b;
uint32_t batch_stride_d;
uint32_t fusion_flags;
uint32_t base_work_group_z;
uint32_t ne02;
uint32_t ne12;
uint32_t broadcast2;
uint32_t broadcast3;
uint32_t pair_m0;
uint32_t stride_d2;
uint32_t batch_stride_d2;
uint32_t lora_r;
float lora_scale0;
float lora_scale1;
};

struct vk_mat_vec_p021_push_constants {
Expand Down Expand Up @@ -66,6 +110,16 @@ struct vk_mat_mat_id_push_constants {
uint32_t row_ids_offset;
};

struct vk_mat_mat_id_fused_push_constants {
uint32_t M; uint32_t N; uint32_t K;
uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;
uint32_t n_experts;
uint32_t row_ids_offset;
uint32_t fusion_flags;
};

struct vk_mat_vec_id_push_constants {
uint32_t ncols;
uint32_t stride_a;
Expand Down Expand Up @@ -454,15 +508,23 @@ struct vk_op_multi_add_push_constants {
uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;

// strides for srcs+dst
uint32_t nb[MAX_PARAMETER_COUNT][4];
uint32_t nb[12][4];

uint32_t rms_partials;
};

static_assert(MAX_PARAMETER_COUNT == 12);

static_assert(sizeof(vk_op_multi_add_push_constants) <= 256);

struct vk_op_multi_add_rms_push_constants {
uint32_t ne0;
uint32_t nrows;
float eps;
uint32_t write_add_dst;
uint32_t nb1[10];
};

static_assert(sizeof(vk_op_multi_add_rms_push_constants) <= 128);

struct vk_op_topk_moe_push_constants {
uint32_t n_rows;
uint32_t n_experts_push;
Expand Down Expand Up @@ -730,6 +792,7 @@ struct vk_op_gated_delta_net_push_constants {
uint32_t neq1, rq3;
float scale;
uint32_t K;
uint32_t sout_nb2;
};

struct vk_op_ssm_scan_push_constants {
Expand All @@ -746,6 +809,9 @@ struct vk_op_ssm_conv_push_constants {
uint32_t nb11;
uint32_t dst_nb0, dst_nb1, dst_nb2;
uint32_t nc, ncs, nr, n_t, n_s;
uint32_t st_nb1, st_nb2;
uint32_t qkv_nb0, qkv_nb1, qkv_nb2;
uint32_t st_k, st_slot_nb;
};

struct vk_op_conv2d_push_constants {
Expand Down
Loading