From 5cc0e90bc88d4695d309d4792c86861cd35ba5f6 Mon Sep 17 00:00:00 2001 From: tawei-amd Date: Sat, 31 Oct 2026 17:39:38 -0700 Subject: [PATCH] feat(server): honour ignore_eos and min_tokens on the OpenAI and Ollama endpoints Mask the end-of-sequence ids in AutoModel::_sample() while ignore_eos is set or fewer than min_tokens tokens were generated (fixes #881). DFlash verifies through _sample() so it is covered; the MTP loop skips speculate() while the mask is needed, since its draft head samples inside the engine. Co-Authored-By: Claude Sonnet 5.5 --- .../instructions/server/request_parameters.md | 4 +++ src/common/AutoModel/automodel.cpp | 25 ++++++++++++++++++- src/common/AutoModel/modeling_qwen3_8mtp.cpp | 5 +++- src/include/AutoModel/automodel.hpp | 4 +++ src/server/rest_handler.cpp | 8 ++++++ 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/docs/instructions/server/request_parameters.md b/docs/docs/instructions/server/request_parameters.md index fe32b7349..dafe5463e 100644 --- a/docs/docs/instructions/server/request_parameters.md +++ b/docs/docs/instructions/server/request_parameters.md @@ -38,6 +38,10 @@ This page says what FLM does with the fields of a `/v1/chat/completions` or `/v1 | `repetition_penalty` | > 0 | FLM extension. See [Penalties](#-presence-frequency-and-repetition-penalties). | | `repeat_last_n` | integer ≥ −1 | FLM extension. How many generated tokens the penalties look at. See [The Penalty Window](#-the-penalty-window). | | `seed` | any integer | Makes sampling repeatable. See [`seed`](#-seed-and-repeatable-output). | +| `min_tokens` | integer ≥ 0 | vLLM-style extension. End-of-sequence tokens are not sampled until at least `min_tokens` tokens have been generated; the reply can run one or two tokens over (more with speculative decoding). `max_tokens` still ends the reply. | +| `ignore_eos` | `true` / `false` | vLLM-style extension. End-of-sequence tokens are never sampled, so the reply runs to `max_tokens` (or the context limit). Meant for fixed-length benchmarking. | + +> `min_tokens` and `ignore_eos` mask the end-of-sequence ids before sampling, so they work on every sampling path. `qwen3.8-mtp` decodes one token at a time while either is in force, because its draft head samples inside the engine; expect a much lower speed there. Each token is chosen in this order: penalties → the `tool_choice` / `response_format` grammar, if any → `top_k` → `top_p` and the model's min-p cut-off → `temperature` → sample. diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 7beb98f88..5535f2229 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -722,7 +722,30 @@ std::string AutoModel::token_text(int token) { int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info) { decode_constraint* constraint = meta_info.constraint.get(); - const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); + bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); + // ignore_eos / min_tokens: take the EOS ids out of the allowed set + if (meta_info.ignore_eos || meta_info.generated_tokens < meta_info.min_tokens) { + const size_t words = (logits.size() + 31) / 32; + if (!masked) { + this->constraint_mask_.assign(words, 0xFFFFFFFFu); + } else if (this->constraint_mask_.size() < words) { + this->constraint_mask_.resize(words, 0u); + } + std::vector saved = this->constraint_mask_; + for (int id : this->eos_token_ids) { + if (id >= 0 && static_cast(id) < logits.size()) { + this->constraint_mask_[id / 32] &= ~(1u << (id % 32)); + } + } + // a grammar that only allows EOS here keeps it: nothing else could be sampled + bool any = false; + for (uint32_t w : this->constraint_mask_) { if (w) { any = true; break; } } + if (any) { + masked = true; + } else { + this->constraint_mask_ = saved; + } + } sample_logprobs_t raw; raw.top_n = meta_info.logprobs_top_n; // The mask goes to the sampler, which applies it to its own copy: the logits diff --git a/src/common/AutoModel/modeling_qwen3_8mtp.cpp b/src/common/AutoModel/modeling_qwen3_8mtp.cpp index e4b8158f2..a75ff061f 100644 --- a/src/common/AutoModel/modeling_qwen3_8mtp.cpp +++ b/src/common/AutoModel/modeling_qwen3_8mtp.cpp @@ -568,7 +568,10 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l const int tokens_left = (length_limit > 0) ? length_limit - meta_info.generated_tokens : SPEC_MAX_DRAFT + 1; - if (spec_enabled && tokens_left > 1) { + // speculate() samples inside the engine, past _sample()'s EOS mask: while ignore_eos / min_tokens + // still forbids an EOS, decode one token at a time + const bool eos_suppressed = meta_info.ignore_eos || meta_info.generated_tokens < meta_info.min_tokens; + if (spec_enabled && tokens_left > 1 && !eos_suppressed) { this->profiler_list[DECODING_TIME].start(); std::vector accepted = this->lm_engine->speculate(last_sampled_token, diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index f679567fb..82046dc30 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -111,6 +111,10 @@ struct chat_meta_info_t { bool restore_allowed; /// \brief Restricts sampling for this request (tool_choice, response_format); may be null std::shared_ptr constraint; + /// \brief ignore_eos / min_tokens of the request: end-of-sequence tokens are + /// never sampled (ignore_eos) or not before min_tokens tokens exist + bool ignore_eos = false; + int min_tokens = 0; /// \brief insert() sampled a token that generate() has not used yet /// \note Models that make the model emit preamble tokens in generate() /// throw that sample away; _force_token() takes it back out of the constraint. diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 5332abcd2..5460bad76 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -996,6 +996,8 @@ void RestHandler::handle_generate(const json& request, ? std::optional(request.at("max_tokens").get()) : std::nullopt); meta_info.max_prefill_len = this->prefill_chunk_len; + meta_info.ignore_eos = value_or(request, "ignore_eos", false); + meta_info.min_tokens = value_or(request, "min_tokens", 0); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; header_print("FLM", "Start generating..."); @@ -1122,6 +1124,8 @@ void RestHandler::handle_chat(const json& request, : std::nullopt); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; + meta_info.ignore_eos = value_or(request, "ignore_eos", false); + meta_info.min_tokens = value_or(request, "min_tokens", 0); header_print("FLM", "Start generating..."); if (stream) { // Streaming response using streaming_ostream @@ -1584,6 +1588,8 @@ void RestHandler::handle_openai_chat_completion(const json& request, normalize_requested_max_new_tokens(openai_chat_budget); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; + meta_info.ignore_eos = value_or(request, "ignore_eos", false); + meta_info.min_tokens = value_or(request, "min_tokens", 0); meta_info.logprobs_top_n = logprobs_top_n; meta_info.continue_final_message = continue_final; try { @@ -1954,6 +1960,8 @@ void RestHandler::handle_openai_completion(const json& request, chat_meta_info_t meta_info; meta_info.max_prefill_len = this->prefill_chunk_len; + meta_info.ignore_eos = value_or(request, "ignore_eos", false); + meta_info.min_tokens = value_or(request, "min_tokens", 0); lm_uniform_input_t uniformed_input; uniformed_input.requested_max_new_tokens = normalize_requested_max_new_tokens(