(fix): fix to handle non json llm response

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -465,14 +465,49 @@ export async function fetchLLMCompletion(
           ? { method: "functionCalling" as const }
           : undefined;
 
-      const structuredOutput = await (chatModel as ChatOpenAI)
-        .withStructuredOutput(
-          params.structuredOutputSchema,
-          structuredOutputConfig,
-        )
-        .invoke(finalMessages, runConfig);
+      try {
+        const structuredOutput = await (chatModel as ChatOpenAI)
+          .withStructuredOutput(
+            params.structuredOutputSchema,
+            structuredOutputConfig,
+          )
+          .invoke(finalMessages, runConfig);
+
+        return structuredOutput;
+      } catch (structuredErr) {
+        // Fallback: some OpenAI-compatible providers return markdown-wrapped JSON
+        // (e.g. ```json\n{...}\n``` or ## headers) that withStructuredOutput can't parse.
+        // Try invoking the model directly and extracting JSON from the raw response.
+        const errMsg =
+          structuredErr instanceof Error
+            ? structuredErr.message
+            : String(structuredErr);
+
+        if (!errMsg.includes("is not valid JSON")) {
+          throw structuredErr;
+        }
+
+        logger.warn(
+          "Structured output failed with JSON parse error, attempting raw extraction fallback",
+        );
+
+        const rawResponse = await chatModel
+          .pipe(new StringOutputParser())
+          .invoke(finalMessages, runConfig);
+
+        const jsonStr = extractJsonFromLLMResponse(rawResponse);
+        if (!jsonStr) {
+          throw structuredErr;
+        }
+
+        const parsed = JSON.parse(jsonStr);
+        const validated = params.structuredOutputSchema.safeParse(parsed);
+        if (!validated.success) {
+          throw structuredErr;
+        }
 
-      return structuredOutput;
+        return validated.data;
+      }
     }
 
     if (tools && tools.length > 0) {
@@ -649,6 +684,81 @@ function processOpenAIBaseURL(params: {
   return url.replace("{model}", modelName);
 }
 
+/**
+ * Extracts a JSON object from an LLM response that may be wrapped in markdown.
+ * Handles cases like:
+ * - ```json\n{...}\n```
+ * - ## Analysis\n{...}
+ * - Some text before\n{...}\nSome text after
+ * - Raw JSON (returns as-is)
+ */
+export function extractJsonFromLLMResponse(text: string): string | null {
+  const trimmed = text.trim();
+
+  // Already valid JSON
+  try {
+    JSON.parse(trimmed);
+    return trimmed;
+  } catch {
+    // Not raw JSON, continue
+  }
+
+  // Extract from markdown code block (```json ... ``` or ``` ... ```)
+  const codeBlockMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/);
+  if (codeBlockMatch) {
+    try {
+      JSON.parse(codeBlockMatch[1].trim());
+      return codeBlockMatch[1].trim();
+    } catch {
+      // Code block content isn't valid JSON either, continue
+    }
+  }
+
+  // Find the first complete JSON object in the text
+  const firstBrace = trimmed.indexOf("{");
+  if (firstBrace === -1) return null;
+
+  let depth = 0;
+  let inString = false;
+  let escape = false;
+
+  for (let i = firstBrace; i < trimmed.length; i++) {
+    const ch = trimmed[i];
+
+    if (escape) {
+      escape = false;
+      continue;
+    }
+
+    if (ch === "\\") {
+      escape = true;
+      continue;
+    }
+
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) continue;
+
+    if (ch === "{") depth++;
+    if (ch === "}") depth--;
+
+    if (depth === 0) {
+      const candidate = trimmed.slice(firstBrace, i + 1);
+      try {
+        JSON.parse(candidate);
+        return candidate;
+      } catch {
+        return null;
+      }
+    }
+  }
+
+  return null;
+}
+
 function extractCleanErrorMessage(rawMessage: string): string {
   // Try to parse JSON error format (common in Google/Vertex AI errors)
   // Example: '[{"error":{"code":404,"message":"Model not found..."}}]'

worker/src/__tests__/extractJsonFromLLMResponse.unit.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, it } from "vitest";
+import { extractJsonFromLLMResponse } from "@langfuse/shared/src/server";
+
+describe("extractJsonFromLLMResponse", () => {
+  it("returns raw JSON as-is when already valid", () => {
+    const input = '{"score": 9, "reasoning": "All good"}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("returns raw JSON with nested objects", () => {
+    const input = '{"a": {"b": [1, 2, 3]}, "c": true}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("extracts JSON from markdown code block with json tag", () => {
+    const input = '```json\n{"score": 5, "reasoning": "minor error"}\n```';
+    expect(extractJsonFromLLMResponse(input)).toBe(
+      '{"score": 5, "reasoning": "minor error"}',
+    );
+  });
+
+  it("extracts JSON from markdown code block without json tag", () => {
+    const input = '```\n{"score": 3}\n```';
+    expect(extractJsonFromLLMResponse(input)).toBe('{"score": 3}');
+  });
+
+  it("extracts JSON after markdown headers", () => {
+    const input =
+      '## Analysis\n\nHere is my evaluation:\n\n{"detected_phonetic_errors": ["None"], "reasoning": "Clean", "score": 10}';
+    expect(extractJsonFromLLMResponse(input)).toBe(
+      '{"detected_phonetic_errors": ["None"], "reasoning": "Clean", "score": 10}',
+    );
+  });
+
+  it("extracts JSON with text before and after", () => {
+    const input = 'Some preamble text.\n{"score": 7}\nSome trailing text.';
+    expect(extractJsonFromLLMResponse(input)).toBe('{"score": 7}');
+  });
+
+  it("extracts JSON with hash headers (## Analysis)", () => {
+    const input =
+      '## Analysis\n\nThe transcript looks clean.\n\n```json\n{"score": 9}\n```';
+    expect(extractJsonFromLLMResponse(input)).toBe('{"score": 9}');
+  });
+
+  it("handles JSON with escaped quotes inside strings", () => {
+    const input = '{"reasoning": "User said \\"hello\\" and confirmed"}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("handles JSON with braces inside strings (not counting depth)", () => {
+    const input = '{"text": "use {curly} braces"}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("returns null for text with no JSON", () => {
+    const input = "This is just plain text with no JSON at all.";
+    expect(extractJsonFromLLMResponse(input)).toBeNull();
+  });
+
+  it("returns null for empty string", () => {
+    expect(extractJsonFromLLMResponse("")).toBeNull();
+  });
+
+  it("returns null for whitespace only", () => {
+    expect(extractJsonFromLLMResponse("   \n\t  ")).toBeNull();
+  });
+
+  it("returns null for incomplete JSON", () => {
+    const input = '{"score": 9, "reasoning":';
+    expect(extractJsonFromLLMResponse(input)).toBeNull();
+  });
+
+  it("returns null for markdown code block with non-JSON content", () => {
+    const input = "```javascript\nconsole.log('hello');\n```";
+    expect(extractJsonFromLLMResponse(input)).toBeNull();
+  });
+
+  it("extracts first valid JSON object when multiple exist", () => {
+    const input = '{"a": 1} {"b": 2}';
+    expect(extractJsonFromLLMResponse(input)).toBe('{"a": 1}');
+  });
+
+  it("handles JSON with special characters in values", () => {
+    const input =
+      '{"reasoning": "User said यस (yes) in Hindi transliteration"}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("handles JSON with newlines inside string values", () => {
+    const input = '{"reasoning": "line1\\nline2\\nline3", "score": 8}';
+    expect(extractJsonFromLLMResponse(input)).toBe(input);
+  });
+
+  it("extracts from response starting with # (markdown header)", () => {
+    const input =
+      '# Evaluation Result\n\n{"detected_phonetic_errors": [], "score": 10, "reasoning": "No issues"}';
+    expect(extractJsonFromLLMResponse(input)).toBe(
+      '{"detected_phonetic_errors": [], "score": 10, "reasoning": "No issues"}',
+    );
+  });
+
+  it("extracts from response starting with backtick (code fence)", () => {
+    const input = '```json\n{"score": 1, "reasoning": "Severe failure"}\n```';
+    const result = extractJsonFromLLMResponse(input);
+    expect(result).toBe('{"score": 1, "reasoning": "Severe failure"}');
+  });
+
+  it("handles real-world wrapped evaluation response", () => {
+    const input = `## Analysis
+
+The outcome is "CONFIRM" which indicates a successful call.
+Score: 9-10 since the call concluded successfully.
+
+\`\`\`json
+{
+  "detected_phonetic_errors": ["None"],
+  "reasoning": "The outcome is CONFIRM. Successful call.",
+  "score": 9
+}
+\`\`\``;
+    const result = extractJsonFromLLMResponse(input);
+    expect(result).not.toBeNull();
+    const parsed = JSON.parse(result!);
+    expect(parsed.score).toBe(9);
+    expect(parsed.reasoning).toContain("CONFIRM");
+  });
+});