From 20c824ef860422038527983458256f131c83b5a4 Mon Sep 17 00:00:00 2001 From: Narsimha Reddy Date: Mon, 4 May 2026 13:23:06 +0530 Subject: [PATCH] (fix): fix to handle non json llm response - some llm services might return non json or md responses for evlauator output added support to extract json and parse it if it happens --- .../src/server/llm/fetchLLMCompletion.ts | 124 ++++++++++++++++- .../extractJsonFromLLMResponse.unit.test.ts | 128 ++++++++++++++++++ 2 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 worker/src/__tests__/extractJsonFromLLMResponse.unit.test.ts diff --git a/packages/shared/src/server/llm/fetchLLMCompletion.ts b/packages/shared/src/server/llm/fetchLLMCompletion.ts index c65ac9a41fcf..33cd598e0a9b 100644 --- a/packages/shared/src/server/llm/fetchLLMCompletion.ts +++ b/packages/shared/src/server/llm/fetchLLMCompletion.ts @@ -465,14 +465,49 @@ export async function fetchLLMCompletion( ? { method: "functionCalling" as const } : undefined; - const structuredOutput = await (chatModel as ChatOpenAI) - .withStructuredOutput( - params.structuredOutputSchema, - structuredOutputConfig, - ) - .invoke(finalMessages, runConfig); + try { + const structuredOutput = await (chatModel as ChatOpenAI) + .withStructuredOutput( + params.structuredOutputSchema, + structuredOutputConfig, + ) + .invoke(finalMessages, runConfig); + + return structuredOutput; + } catch (structuredErr) { + // Fallback: some OpenAI-compatible providers return markdown-wrapped JSON + // (e.g. ```json\n{...}\n``` or ## headers) that withStructuredOutput can't parse. + // Try invoking the model directly and extracting JSON from the raw response. + const errMsg = + structuredErr instanceof Error + ? structuredErr.message + : String(structuredErr); + + if (!errMsg.includes("is not valid JSON")) { + throw structuredErr; + } + + logger.warn( + "Structured output failed with JSON parse error, attempting raw extraction fallback", + ); + + const rawResponse = await chatModel + .pipe(new StringOutputParser()) + .invoke(finalMessages, runConfig); + + const jsonStr = extractJsonFromLLMResponse(rawResponse); + if (!jsonStr) { + throw structuredErr; + } + + const parsed = JSON.parse(jsonStr); + const validated = params.structuredOutputSchema.safeParse(parsed); + if (!validated.success) { + throw structuredErr; + } - return structuredOutput; + return validated.data; + } } if (tools && tools.length > 0) { @@ -649,6 +684,81 @@ function processOpenAIBaseURL(params: { return url.replace("{model}", modelName); } +/** + * Extracts a JSON object from an LLM response that may be wrapped in markdown. + * Handles cases like: + * - ```json\n{...}\n``` + * - ## Analysis\n{...} + * - Some text before\n{...}\nSome text after + * - Raw JSON (returns as-is) + */ +export function extractJsonFromLLMResponse(text: string): string | null { + const trimmed = text.trim(); + + // Already valid JSON + try { + JSON.parse(trimmed); + return trimmed; + } catch { + // Not raw JSON, continue + } + + // Extract from markdown code block (```json ... ``` or ``` ... ```) + const codeBlockMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/); + if (codeBlockMatch) { + try { + JSON.parse(codeBlockMatch[1].trim()); + return codeBlockMatch[1].trim(); + } catch { + // Code block content isn't valid JSON either, continue + } + } + + // Find the first complete JSON object in the text + const firstBrace = trimmed.indexOf("{"); + if (firstBrace === -1) return null; + + let depth = 0; + let inString = false; + let escape = false; + + for (let i = firstBrace; i < trimmed.length; i++) { + const ch = trimmed[i]; + + if (escape) { + escape = false; + continue; + } + + if (ch === "\\") { + escape = true; + continue; + } + + if (ch === '"') { + inString = !inString; + continue; + } + + if (inString) continue; + + if (ch === "{") depth++; + if (ch === "}") depth--; + + if (depth === 0) { + const candidate = trimmed.slice(firstBrace, i + 1); + try { + JSON.parse(candidate); + return candidate; + } catch { + return null; + } + } + } + + return null; +} + function extractCleanErrorMessage(rawMessage: string): string { // Try to parse JSON error format (common in Google/Vertex AI errors) // Example: '[{"error":{"code":404,"message":"Model not found..."}}]' diff --git a/worker/src/__tests__/extractJsonFromLLMResponse.unit.test.ts b/worker/src/__tests__/extractJsonFromLLMResponse.unit.test.ts new file mode 100644 index 000000000000..7a8f613c2138 --- /dev/null +++ b/worker/src/__tests__/extractJsonFromLLMResponse.unit.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { extractJsonFromLLMResponse } from "@langfuse/shared/src/server"; + +describe("extractJsonFromLLMResponse", () => { + it("returns raw JSON as-is when already valid", () => { + const input = '{"score": 9, "reasoning": "All good"}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("returns raw JSON with nested objects", () => { + const input = '{"a": {"b": [1, 2, 3]}, "c": true}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("extracts JSON from markdown code block with json tag", () => { + const input = '```json\n{"score": 5, "reasoning": "minor error"}\n```'; + expect(extractJsonFromLLMResponse(input)).toBe( + '{"score": 5, "reasoning": "minor error"}', + ); + }); + + it("extracts JSON from markdown code block without json tag", () => { + const input = '```\n{"score": 3}\n```'; + expect(extractJsonFromLLMResponse(input)).toBe('{"score": 3}'); + }); + + it("extracts JSON after markdown headers", () => { + const input = + '## Analysis\n\nHere is my evaluation:\n\n{"detected_phonetic_errors": ["None"], "reasoning": "Clean", "score": 10}'; + expect(extractJsonFromLLMResponse(input)).toBe( + '{"detected_phonetic_errors": ["None"], "reasoning": "Clean", "score": 10}', + ); + }); + + it("extracts JSON with text before and after", () => { + const input = 'Some preamble text.\n{"score": 7}\nSome trailing text.'; + expect(extractJsonFromLLMResponse(input)).toBe('{"score": 7}'); + }); + + it("extracts JSON with hash headers (## Analysis)", () => { + const input = + '## Analysis\n\nThe transcript looks clean.\n\n```json\n{"score": 9}\n```'; + expect(extractJsonFromLLMResponse(input)).toBe('{"score": 9}'); + }); + + it("handles JSON with escaped quotes inside strings", () => { + const input = '{"reasoning": "User said \\"hello\\" and confirmed"}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("handles JSON with braces inside strings (not counting depth)", () => { + const input = '{"text": "use {curly} braces"}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("returns null for text with no JSON", () => { + const input = "This is just plain text with no JSON at all."; + expect(extractJsonFromLLMResponse(input)).toBeNull(); + }); + + it("returns null for empty string", () => { + expect(extractJsonFromLLMResponse("")).toBeNull(); + }); + + it("returns null for whitespace only", () => { + expect(extractJsonFromLLMResponse(" \n\t ")).toBeNull(); + }); + + it("returns null for incomplete JSON", () => { + const input = '{"score": 9, "reasoning":'; + expect(extractJsonFromLLMResponse(input)).toBeNull(); + }); + + it("returns null for markdown code block with non-JSON content", () => { + const input = "```javascript\nconsole.log('hello');\n```"; + expect(extractJsonFromLLMResponse(input)).toBeNull(); + }); + + it("extracts first valid JSON object when multiple exist", () => { + const input = '{"a": 1} {"b": 2}'; + expect(extractJsonFromLLMResponse(input)).toBe('{"a": 1}'); + }); + + it("handles JSON with special characters in values", () => { + const input = + '{"reasoning": "User said यस (yes) in Hindi transliteration"}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("handles JSON with newlines inside string values", () => { + const input = '{"reasoning": "line1\\nline2\\nline3", "score": 8}'; + expect(extractJsonFromLLMResponse(input)).toBe(input); + }); + + it("extracts from response starting with # (markdown header)", () => { + const input = + '# Evaluation Result\n\n{"detected_phonetic_errors": [], "score": 10, "reasoning": "No issues"}'; + expect(extractJsonFromLLMResponse(input)).toBe( + '{"detected_phonetic_errors": [], "score": 10, "reasoning": "No issues"}', + ); + }); + + it("extracts from response starting with backtick (code fence)", () => { + const input = '```json\n{"score": 1, "reasoning": "Severe failure"}\n```'; + const result = extractJsonFromLLMResponse(input); + expect(result).toBe('{"score": 1, "reasoning": "Severe failure"}'); + }); + + it("handles real-world wrapped evaluation response", () => { + const input = `## Analysis + +The outcome is "CONFIRM" which indicates a successful call. +Score: 9-10 since the call concluded successfully. + +\`\`\`json +{ + "detected_phonetic_errors": ["None"], + "reasoning": "The outcome is CONFIRM. Successful call.", + "score": 9 +} +\`\`\``; + const result = extractJsonFromLLMResponse(input); + expect(result).not.toBeNull(); + const parsed = JSON.parse(result!); + expect(parsed.score).toBe(9); + expect(parsed.reasoning).toContain("CONFIRM"); + }); +});