Add exact request budget preflight

sdk_v2/cpp/CMakeLists.txt

@@ -214,12 +214,14 @@ set(FOUNDRY_LOCAL_SOURCES
     src/inferencing/generative/embeddings/embeddings_session.cc
     src/inferencing/generative/chat/chat_generator.cc
     src/inferencing/session/session.cc
+    src/inferencing/session/request.cc
     src/inferencing/session/session_manager.cc
     src/inferencing/session/tool_registry.cc
     src/inferencing/generative/chat/chat_session.cc
     src/inferencing/generative/chat/chat_template.cc
     src/inferencing/generative/chat/chat_transcript.cc
     src/inferencing/generative/chat/media_input.cc
+    src/inferencing/generative/chat/prepared_chat_prompt.cc
     src/configuration.cc
     src/download/blob_download_state.cc
     src/download/blob_downloader.cc

sdk_v2/cpp/include/foundry_local/foundry_local_c.h

@@ -126,6 +126,7 @@ FL_TYPE(ModelList);
 // Request accumulates parameters and input items
 // Response accumulates output items
 FL_TYPE(Request);
+FL_TYPE(RequestPreflight);
 FL_TYPE(Response);
 
 // Opaque type for a session. Create with loaded Model so Model:Session is 1:M
@@ -377,6 +378,18 @@ typedef struct flUsage {
   /* V3 fields go here. Read only when version >= 3. */
 } flUsage;
 
+/// Exact token-budget preflight result for a request in the captured session state.
+typedef struct flRequestPreflightResult {
+  uint32_t version;               ///< Set to FOUNDRY_LOCAL_API_VERSION.
+  int64_t prompt_tokens;          ///< Exact number of prompt tokens after request preparation.
+  int64_t output_reserve_tokens;  ///< Tokens reserved for generated output.
+  int64_t required_tokens;        ///< Total tokens required: prompt plus output reserve.
+  int64_t context_limit_tokens;   ///< Model context-window limit.
+  bool fits;                      ///< Whether required_tokens fits within the context limit.
+  int64_t deficit_tokens;         ///< Tokens over budget, or 0 when fits is true.
+  /* V3 fields go here. Read only when version >= 3. */
+} flRequestPreflightResult;
+
 /// Information about a discoverable execution provider.
 /// Returned by Manager_GetDiscoverableEps. Storage is owned by the Manager; the
 /// returned pointers and `name` strings are stable for the Manager's lifetime.
@@ -953,6 +966,21 @@ struct flInferenceApi {
   FL_API_STATUS(Session_UndoTurns, _In_ flSession* session, size_t count);
 
   // End V1
+
+  /// Capture the request and current chat-session state for an exact token-budget preflight.
+  /// The returned one-shot handle remains valid after the source request and session are released. The Manager and
+  /// its model runtime must remain alive until the handle is executed and released. Execute may run on another
+  /// thread; callers must not execute and release the same handle concurrently.
+  FL_API_STATUS(Session_CreateRequestPreflight, _In_ const flSession* session, _In_ const flRequest* request,
+                _Outptr_ flRequestPreflight** out_preflight);
+
+  /// Synchronously execute a preflight. This operation may be expensive.
+  /// The caller must initialize out_result->version to FOUNDRY_LOCAL_API_VERSION.
+  FL_API_STATUS(RequestPreflight_Execute, _In_ flRequestPreflight* preflight,
+                _Out_ flRequestPreflightResult* out_result);
+
+  /// Release a preflight handle. Passing nullptr is allowed.
+  FL_TYPE_RELEASE(RequestPreflight);
 };
 
 /* --- Configuration API ------------------------------------------------- */

sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h

@@ -1037,6 +1037,29 @@ class Request {
   detail::Base<flRequest> handle_;
 };
 
+/// Captured exact token-budget preflight operation.
+///
+/// Capture is separate from execution so callers can capture request and session state on one
+/// thread, then move the operation to a worker for the potentially expensive execution. The
+/// source Request and ChatSession do not need to remain alive after capture. The owning Manager
+/// and model runtime must remain alive until this operation is executed and destroyed.
+class RequestPreflight {
+ public:
+  RequestPreflight(const RequestPreflight&) = delete;
+  RequestPreflight& operator=(const RequestPreflight&) = delete;
+  RequestPreflight(RequestPreflight&&) noexcept = default;
+  RequestPreflight& operator=(RequestPreflight&&) noexcept = default;
+
+  /// Execute the captured preflight operation.
+  flRequestPreflightResult Execute();
+
+ private:
+  friend class ChatSession;
+  explicit RequestPreflight(flRequestPreflight* preflight);
+
+  detail::Base<flRequestPreflight> handle_;
+};
+
 /// Wrapper for an opaque flResponse.
 class Response {
  public:
@@ -1104,6 +1127,16 @@ class ChatSession : public Session {
   /// Undo the last `count` turns and remove their input messages and assistant replies from history.
   /// Retained inference state is reused when it can be restored safely; otherwise it is rebuilt on the next request.
   void UndoTurns(size_t count);
+
+  /// Capture request and session state for a later exact token-budget preflight.
+  /// The returned operation can be moved to a worker and executed after the source request and
+  /// session are destroyed. The owning Manager and model runtime must remain alive until the
+  /// returned operation is executed and destroyed.
+  RequestPreflight CaptureRequestPreflight(const Request& request) const;
+
+  /// Synchronously capture and execute an exact token-budget preflight without mutating the request or session.
+  /// This operation may be expensive. The owning Manager and model runtime must remain alive for the call.
+  flRequestPreflightResult PreflightRequest(const Request& request) const;
 };
 
 /// Session for automatic-speech-recognition (transcription) models.

sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h

@@ -1160,6 +1160,20 @@ inline flRequest* detail::CreateRequest() {
   return req;
 }
 
+// ===========================================================================
+// RequestPreflight
+// ===========================================================================
+
+inline RequestPreflight::RequestPreflight(flRequestPreflight* preflight)
+    : handle_(preflight, detail::inference_api()->RequestPreflight_Release) {}
+
+inline flRequestPreflightResult RequestPreflight::Execute() {
+  flRequestPreflightResult result{};
+  result.version = FOUNDRY_LOCAL_API_VERSION;
+  Check(detail::inference_api()->RequestPreflight_Execute(handle_.get_mutable(), &result));
+  return result;
+}
+
 // ===========================================================================
 // Response
 // ===========================================================================
@@ -1270,6 +1284,16 @@ inline void ChatSession::UndoTurns(size_t count) {
   Check(detail::inference_api()->Session_UndoTurns(handle_.get_mutable(), count));
 }
 
+inline RequestPreflight ChatSession::CaptureRequestPreflight(const Request& request) const {
+  flRequestPreflight* preflight = nullptr;
+  Check(detail::inference_api()->Session_CreateRequestPreflight(handle_.get(), request.native_handle(), &preflight));
+  return RequestPreflight(preflight);
+}
+
+inline flRequestPreflightResult ChatSession::PreflightRequest(const Request& request) const {
+  return CaptureRequestPreflight(request).Execute();
+}
+
 // ===========================================================================
 // AudioSession
 // ===========================================================================

sdk_v2/cpp/src/c_api.cc

@@ -1903,6 +1903,51 @@ FL_API_STATUS_IMPL(Session_UndoTurnsImpl, flSession* session, size_t count) {
   API_IMPL_END
 }
 
+FL_API_STATUS_IMPL(Session_CreateRequestPreflightImpl, const flSession* session, const flRequest* request,
+                   flRequestPreflight** out_preflight) {
+  API_IMPL_BEGIN
+  if (!out_preflight) {
+    return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null out_preflight");
+  }
+
+  *out_preflight = nullptr;
+  if (!session || !request) {
+    return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument");
+  }
+
+  auto operation = AsImpl(session)->CreateRequestPreflight(*AsImpl(request));
+  *out_preflight = reinterpret_cast<flRequestPreflight*>(operation.release());
+  return nullptr;
+  API_IMPL_END
+}
+
+FL_API_STATUS_IMPL(RequestPreflight_ExecuteImpl, flRequestPreflight* preflight,
+                   flRequestPreflightResult* out_result) {
+  API_IMPL_BEGIN
+  if (!preflight || !out_result) {
+    return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument");
+  }
+  if (out_result->version != FOUNDRY_LOCAL_API_VERSION) {
+    return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "out_result->version is not supported");
+  }
+
+  auto* operation = reinterpret_cast<fl::Session::RequestPreflightOperation*>(preflight);
+  const auto result = operation->Execute();
+  out_result->version = FOUNDRY_LOCAL_API_VERSION;
+  out_result->prompt_tokens = result.prompt_tokens;
+  out_result->output_reserve_tokens = result.output_reserve_tokens;
+  out_result->required_tokens = result.required_tokens;
+  out_result->context_limit_tokens = result.context_limit_tokens;
+  out_result->fits = result.fits;
+  out_result->deficit_tokens = result.deficit_tokens;
+  return nullptr;
+  API_IMPL_END
+}
+
+static void FL_API_CALL RequestPreflight_ReleaseImpl(flRequestPreflight* preflight) FL_NO_EXCEPTION {
+  delete reinterpret_cast<fl::Session::RequestPreflightOperation*>(preflight);
+}
+
 static const flInferenceApi g_inference_api = {
     Request_CreateImpl,
     Request_ReleaseImpl,
@@ -1926,10 +1971,18 @@ static const flInferenceApi g_inference_api = {
     Session_RemoveToolDefinitionImpl,
     Session_GetTurnCountImpl,
     Session_UndoTurnsImpl,
+    Session_CreateRequestPreflightImpl,
+    RequestPreflight_ExecuteImpl,
+    RequestPreflight_ReleaseImpl,
 };
 
 static_assert(offsetof(flInferenceApi, Session_UndoTurns) / sizeof(void*) == 21,
-              "Size of version 1 Inference API cannot change");
+              "Version 1 Inference API prefix cannot change");
+static_assert(offsetof(flInferenceApi, Session_CreateRequestPreflight) / sizeof(void*) == 22,
+              "Version 2 Inference API must append after the version 1 prefix");
+static_assert(sizeof(flInferenceApi) ==
+                  offsetof(flInferenceApi, Session_CreateRequestPreflight) + 3 * sizeof(void*),
+              "Version 2 Inference API must append exactly three preflight functions");
 
 // ========================================================================
 // Sub-API accessors

sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc

@@ -3,11 +3,20 @@
 #include "inferencing/generative/chat/chat_generator.h"
 
 #include "exception.h"
+#include "inferencing/generative/chat/prepared_chat_prompt.h"
 
 namespace fl {
 
 void ChatGenerator::Close() {}
 
+int ChatGenerator::AppendPreparedPrompt(const std::vector<TranscriptMessage>&,
+                                        const PreparedChatPrompt&,
+                                        GenAIModelInstance&,
+                                        const ToolCallContext&,
+                                        const SearchOptions&) {
+  FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "generator does not support prepared prompt append");
+}
+
 void ChatGenerator::RewindTo(int /*token_count*/) {
   FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "This generator does not support rewinding retained model state");
 }

sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h

@@ -13,6 +13,7 @@
 namespace fl {
 
 class GenAIModelInstance;
+struct PreparedChatPrompt;
 struct TranscriptMessage;
 struct SearchOptions;
 struct ToolCallContext;
@@ -96,6 +97,13 @@ class ChatGenerator {
                              const ToolCallContext& tool_ctx,
                              const SearchOptions& options) = 0;
 
+  /// Append a prompt already rendered and tokenized by the authoritative request preparation path.
+  virtual int AppendPreparedPrompt(const std::vector<TranscriptMessage>& new_messages,
+                                   const PreparedChatPrompt& prompt,
+                                   GenAIModelInstance& model,
+                                   const ToolCallContext& tool_ctx,
+                                   const SearchOptions& options);
+
   /// Whether the prompt for the active turn ends inside a reasoning block opened by the chat template.
   virtual bool PromptOpensReasoning() const { return false; }