Feat/prefer streaming

src/core/agent-as-tool.ts

@@ -86,6 +86,7 @@ export function agentAsTool<Ctx, Out = any>(
         maxTurns,
         // Memory isolation by default
         ...(memoryMode === 'none' ? { memory: undefined, conversationId: undefined } : {}),
+        preferStreaming: parentConfig.preferStreaming,
         onEvent: (event) => {
           if (propagateEvents === 'all') {
             parentConfig.onEvent?.(event);

src/core/engine.ts

@@ -496,10 +496,12 @@ async function runInternal<Ctx, Out>(
         llmResponse = await config.modelProvider.getCompletion(state, currentAgent, config);
       }
     } else {
-      if (typeof config.modelProvider.getCompletionStream === 'function') {
+      const getStream = config.modelProvider.getCompletionStream;
+      const useStreaming = config.preferStreaming !== false && typeof getStream === 'function';
+      if (useStreaming) {
         try {
           streamingUsed = true;
-          const stream = config.modelProvider.getCompletionStream(state, currentAgent, config);
+          const stream = getStream(state, currentAgent, config);
           let aggregatedText = '';
           const toolCalls: Array<{ id?: string; type: 'function'; function: { name?: string; arguments: string } }> = [];
 
@@ -569,10 +571,12 @@ async function runInternal<Ctx, Out>(
       }
     }
   } else {
-    if (typeof config.modelProvider.getCompletionStream === 'function') {
+    const getStream = config.modelProvider.getCompletionStream;
+    const useStreaming = config.preferStreaming !== false && typeof getStream === 'function';
+    if (useStreaming) {
       try {
         streamingUsed = true;
-        const stream = config.modelProvider.getCompletionStream(state, currentAgent, config);
+        const stream = getStream(state, currentAgent, config);
         let aggregatedText = '';
         const toolCalls: Array<{ id?: string; type: 'function'; function: { name?: string; arguments: string } }> = [];
 

src/core/guardrails.ts

@@ -267,7 +267,8 @@ ${sanitizedContent}
         modelOverride: modelToUse,
         initialInputGuardrails: undefined,
         finalOutputGuardrails: undefined,
-        onEvent: undefined
+        onEvent: undefined,
+        preferStreaming: config.preferStreaming
       };
 
       const completionPromise = config.modelProvider.getCompletion(tempState, evalAgent, guardrailConfig);

src/core/types.ts

@@ -427,6 +427,7 @@ export type RunConfig<Ctx> = {
   readonly conversationId?: string;
   readonly approvalStorage?: ApprovalStorage;
   readonly defaultFastModel?: string;
+  readonly preferStreaming?: boolean;
 };
 
 export const jsonParseLLMOutput = (text: string): any => {

src/providers/model.ts

@@ -74,9 +74,14 @@ export const makeLiteLLMProvider = <Ctx>(
     async getCompletion(state, agent, config) {
       const { model, params } = await buildChatCompletionParams(state, agent, config, baseURL);
 
-      safeConsole.log(`📞 Calling model: ${model} with params: ${JSON.stringify(params, null, 2)}`);
+      const requestParams = {
+        ...params,
+        stream: false,
+      };
+
+      safeConsole.log(`📞 Calling model: ${model} with params: ${JSON.stringify(requestParams, null, 2)}`);
       const resp = await client.chat.completions.create(
-        params as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming
+        requestParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming
       );
 
       // Return the choice with usage data attached for tracing