feat(server): honour ignore_eos and min_tokens (#881)

docs/docs/instructions/server/request_parameters.md

@@ -38,6 +38,10 @@ This page says what FLM does with the fields of a `/v1/chat/completions` or `/v1
 | `repetition_penalty` | > 0 | FLM extension. See [Penalties](#-presence-frequency-and-repetition-penalties). |
 | `repeat_last_n` | integer ≥ −1 | FLM extension. How many generated tokens the penalties look at. See [The Penalty Window](#-the-penalty-window). |
 | `seed` | any integer | Makes sampling repeatable. See [`seed`](#-seed-and-repeatable-output). |
+| `min_tokens` | integer ≥ 0 | vLLM-style extension. End-of-sequence tokens are not sampled until at least `min_tokens` tokens have been generated; the reply can run one or two tokens over (more with speculative decoding). `max_tokens` still ends the reply. |
+| `ignore_eos` | `true` / `false` | vLLM-style extension. End-of-sequence tokens are never sampled, so the reply runs to `max_tokens` (or the context limit). Meant for fixed-length benchmarking. |
+
+> `min_tokens` and `ignore_eos` mask the end-of-sequence ids before sampling, so they work on every sampling path. `qwen3.8-mtp` decodes one token at a time while either is in force, because its draft head samples inside the engine; expect a much lower speed there.
 
 Each token is chosen in this order: penalties → the `tool_choice` / `response_format` grammar, if any → `top_k` → `top_p` and the model's min-p cut-off → `temperature` → sample.
 

src/common/AutoModel/automodel.cpp

@@ -722,7 +722,30 @@ std::string AutoModel::token_text(int token) {
 
 int AutoModel::_sample(buffer<bf16>& logits, chat_meta_info_t& meta_info) {
     decode_constraint* constraint = meta_info.constraint.get();
-    const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_);
+    bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_);
+    // ignore_eos / min_tokens: take the EOS ids out of the allowed set
+    if (meta_info.ignore_eos || meta_info.generated_tokens < meta_info.min_tokens) {
+        const size_t words = (logits.size() + 31) / 32;
+        if (!masked) {
+            this->constraint_mask_.assign(words, 0xFFFFFFFFu);
+        } else if (this->constraint_mask_.size() < words) {
+            this->constraint_mask_.resize(words, 0u);
+        }
+        std::vector<uint32_t> saved = this->constraint_mask_;
+        for (int id : this->eos_token_ids) {
+            if (id >= 0 && static_cast<size_t>(id) < logits.size()) {
+                this->constraint_mask_[id / 32] &= ~(1u << (id % 32));
+            }
+        }
+        // a grammar that only allows EOS here keeps it: nothing else could be sampled
+        bool any = false;
+        for (uint32_t w : this->constraint_mask_) { if (w) { any = true; break; } }
+        if (any) {
+            masked = true;
+        } else {
+            this->constraint_mask_ = saved;
+        }
+    }
     sample_logprobs_t raw;
     raw.top_n = meta_info.logprobs_top_n;
     // The mask goes to the sampler, which applies it to its own copy: the logits

src/common/AutoModel/modeling_qwen3_8mtp.cpp

@@ -568,7 +568,10 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l
         const int tokens_left = (length_limit > 0)
             ? length_limit - meta_info.generated_tokens
             : SPEC_MAX_DRAFT + 1;
-        if (spec_enabled && tokens_left > 1) {
+        // speculate() samples inside the engine, past _sample()'s EOS mask: while ignore_eos / min_tokens
+        // still forbids an EOS, decode one token at a time
+        const bool eos_suppressed = meta_info.ignore_eos || meta_info.generated_tokens < meta_info.min_tokens;
+        if (spec_enabled && tokens_left > 1 && !eos_suppressed) {
             this->profiler_list[DECODING_TIME].start();
             std::vector<int> accepted =
                 this->lm_engine->speculate(last_sampled_token,

src/include/AutoModel/automodel.hpp

@@ -111,6 +111,10 @@ struct chat_meta_info_t {
 	bool restore_allowed;
 	/// \brief Restricts sampling for this request (tool_choice, response_format); may be null
 	std::shared_ptr<decode_constraint> constraint;
+	/// \brief ignore_eos / min_tokens of the request: end-of-sequence tokens are
+	///        never sampled (ignore_eos) or not before min_tokens tokens exist
+	bool ignore_eos = false;
+	int min_tokens = 0;
 	/// \brief insert() sampled a token that generate() has not used yet
 	/// \note Models that make the model emit preamble tokens in generate()
 	///       throw that sample away; _force_token() takes it back out of the constraint.

src/server/rest_handler.cpp

@@ -996,6 +996,8 @@ void RestHandler::handle_generate(const json& request,
                     ? std::optional<int>(request.at("max_tokens").get<int>())
                     : std::nullopt);
         meta_info.max_prefill_len = this->prefill_chunk_len;
+        meta_info.ignore_eos = value_or(request, "ignore_eos", false);
+        meta_info.min_tokens = value_or(request, "min_tokens", 0);
         meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first;
         header_print("FLM", "Start generating...");
 
@@ -1122,6 +1124,8 @@ void RestHandler::handle_chat(const json& request,
                     : std::nullopt);
         meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first;
         meta_info.max_prefill_len = this->prefill_chunk_len;
+        meta_info.ignore_eos = value_or(request, "ignore_eos", false);
+        meta_info.min_tokens = value_or(request, "min_tokens", 0);
         header_print("FLM", "Start generating...");
         if (stream) {
             // Streaming response using streaming_ostream
@@ -1584,6 +1588,8 @@ void RestHandler::handle_openai_chat_completion(const json& request,
             normalize_requested_max_new_tokens(openai_chat_budget);
         meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first;
         meta_info.max_prefill_len = this->prefill_chunk_len;
+        meta_info.ignore_eos = value_or(request, "ignore_eos", false);
+        meta_info.min_tokens = value_or(request, "min_tokens", 0);
         meta_info.logprobs_top_n = logprobs_top_n;
         meta_info.continue_final_message = continue_final;
         try {
@@ -1954,6 +1960,8 @@ void RestHandler::handle_openai_completion(const json& request,
 
         chat_meta_info_t meta_info;
         meta_info.max_prefill_len = this->prefill_chunk_len;
+        meta_info.ignore_eos = value_or(request, "ignore_eos", false);
+        meta_info.min_tokens = value_or(request, "min_tokens", 0);
         lm_uniform_input_t uniformed_input;
         uniformed_input.requested_max_new_tokens =
             normalize_requested_max_new_tokens(