diff --git a/src/commands/run.ts b/src/commands/run.ts index 6816d3e..c4808ad 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -245,7 +245,7 @@ export const runCommand = new Command('run') ? normalizeProvider(options.analyzerProvider || provider) : ''; const analyzerApiKey = analyze - ? (options.analyzerKey || getApiKey(analyzerProvider) || resolvedApiKey) + ? (options.analyzerKey || getApiKey(analyzerProvider) || (normalizeProvider(provider) === analyzerProvider ? resolvedApiKey : undefined)) : undefined; const analyzerBaseUrl = analyze ? (options.analyzerUrl || getEffectiveProviderUrl(analyzerProvider) || undefined) diff --git a/src/lib/analyzer.ts b/src/lib/analyzer.ts index b80e32c..47fb2b4 100644 --- a/src/lib/analyzer.ts +++ b/src/lib/analyzer.ts @@ -232,18 +232,39 @@ Return ONLY the JSON object, no other text.`; // Response Parser // ============================================================================= +export function extractJson(text: string): string { + let s = text.trim(); + + // Strip markdown fences anywhere in the response + s = s.replace(/^```json\s*/i, '').replace(/^```\s*/, '').replace(/\s*```\s*$/, '').trim(); + + // Find the outermost { ... } — always run the brace scanner to strip trailing text + const start = s.startsWith('{') ? 0 : s.indexOf('{'); + if (start === -1) return s; + let depth = 0; + let inString = false; + let escape = false; + for (let i = start; i < s.length; i++) { + const ch = s[i]; + if (escape) { escape = false; continue; } + if (ch === '\\' && inString) { escape = true; continue; } + if (ch === '"' && !escape) { inString = !inString; continue; } + if (inString) continue; + if (ch === '{') depth++; + else if (ch === '}') { depth--; if (depth === 0) return s.substring(start, i + 1); } + } + // Unclosed — return from first '{' to end (let JSON.parse report the real error) + return s.substring(start); +} + export async function parseAnalysisResponse( response: string, runId: string, result: RunResult, - challengeConfig?: ChallengeConfig + challengeConfig?: ChallengeConfig, + analyzerModel?: string, ): Promise { - let jsonStr = response.trim(); - - if (jsonStr.startsWith('```json')) jsonStr = jsonStr.slice(7); - else if (jsonStr.startsWith('```')) jsonStr = jsonStr.slice(3); - if (jsonStr.endsWith('```')) jsonStr = jsonStr.slice(0, -3); - jsonStr = jsonStr.trim(); + const jsonStr = extractJson(response); try { const parsed = AnalysisResponseSchema.parse(JSON.parse(jsonStr)); @@ -251,7 +272,7 @@ export async function parseAnalysisResponse( const analysisResult: AnalysisResult = { runId, analyzedAt: new Date(), - analyzerModel: DEFAULT_ANALYZER_MODEL, + analyzerModel: analyzerModel || DEFAULT_ANALYZER_MODEL, attackChain: { phases: parsed.attackChain.phases, techniques: parsed.attackChain.techniques, @@ -342,7 +363,7 @@ export async function parseAnalysisResponse( return { runId, analyzedAt: new Date(), - analyzerModel: DEFAULT_ANALYZER_MODEL, + analyzerModel: analyzerModel || DEFAULT_ANALYZER_MODEL, parseFailed: true, attackChain: { phases: [], techniques: [], killChainCoverage: [] }, narrative: { summary: 'Analysis parsing failed', detailed: `Error: ${error}`, keyFindings: [] }, @@ -447,10 +468,21 @@ export function resolveDefaultAnalyzerModel(analyzerProvider: string, benchmarkR const preset = resolveProvider(analyzerProvider); - // Same provider as benchmark — use the benchmark model since we know it's available + // Same provider as benchmark — use the benchmark model (user's choice), but + // filter out known non-text models that can't do chat completions. const benchmarkProvider = normalizeProvider(benchmarkResult.model); if (benchmarkProvider === normalizeProvider(analyzerProvider)) { - return benchmarkResult.modelVersion || preset?.models[0] || DEFAULT_ANALYZER_MODEL; + const NON_TEXT_MODEL_PATTERNS: RegExp[] = [ + /^grok-imagine-/i, + /^dall-e-/i, + /^tts-/i, + /^whisper-/i, + /[-.]embed(ding)?s?[-.:]/i, + /^text-embed(ding)?/i, + ]; + const modelName = benchmarkResult.modelVersion || ''; + const isTextModel = !NON_TEXT_MODEL_PATTERNS.some(p => p.test(modelName)); + return (isTextModel ? benchmarkResult.modelVersion : null) || preset?.models[0] || DEFAULT_ANALYZER_MODEL; } // Different provider — try preset default, fall back to Claude @@ -543,7 +575,7 @@ export async function analyzeRun( responseText = await callOpenAIAnalyzer(apiKey, baseUrl, analyzerModel, prompt); } - return parseAnalysisResponse(responseText, result.id, result, options.challengeConfig); + return parseAnalysisResponse(responseText, result.id, result, options.challengeConfig, analyzerModel); } export async function analyzeExistingRun( diff --git a/src/lib/export.ts b/src/lib/export.ts index fd48500..dc636a6 100644 --- a/src/lib/export.ts +++ b/src/lib/export.ts @@ -14,21 +14,31 @@ export async function promptExport( ): Promise { console.log(); if (!analysis) { - console.log(colors.gray(' No analysis available — export will not include scores.')); + console.log(colors.gray(' No analysis available — skipping export.')); + console.log(colors.gray(` Run analysis first: oasis analyze ${result.id}`)); + console.log(colors.gray(` Then export via: oasis report ${result.id}`)); + return; } console.log(colors.gray(` More export formats available via: oasis report ${result.id}`)); + let hasActed = false; while (true) { + const exportChoices = [ + { name: 'Copy share card to clipboard', value: 'share' as const }, + { name: 'Save HTML report', value: 'html' as const }, + ]; + const doneChoice = { name: hasActed ? 'Done' : colors.gray('Done'), value: 'done' as const }; + const choices = hasActed + ? [doneChoice, ...exportChoices] + : [...exportChoices, doneChoice]; + const action = await select({ message: 'Share results?', - choices: [ - { name: 'Copy share card to clipboard', value: 'share' as const }, - { name: 'Save HTML report', value: 'html' as const }, - { name: colors.gray('Done'), value: 'done' as const }, - ], + choices, }); if (action === 'done') break; + hasActed = true; if (action === 'share') { const card = generateShareCard(result, analysis, ksmScore); diff --git a/tests/unit/analyzer.test.ts b/tests/unit/analyzer.test.ts index 41a0bd5..578a4e7 100644 --- a/tests/unit/analyzer.test.ts +++ b/tests/unit/analyzer.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { calculateKSM, fallbackOverallScore } from '../../src/lib/scoring.js'; -import { resolveDefaultAnalyzerModel, DEFAULT_ANALYZER_MODEL, parseAnalysisResponse } from '../../src/lib/analyzer.js'; +import { resolveDefaultAnalyzerModel, DEFAULT_ANALYZER_MODEL, parseAnalysisResponse, extractJson } from '../../src/lib/analyzer.js'; import type { RunResult } from '../../src/lib/types.js'; function makeRunResult(model: string, modelVersion: string): RunResult { @@ -144,6 +144,26 @@ describe('resolveDefaultAnalyzerModel', () => { expect(resolveDefaultAnalyzerModel('xai', result)).toBe('grok-3-latest'); }); + it('filters out image models — falls back to preset default', () => { + const result = makeRunResult('xai', 'grok-imagine-image'); + expect(resolveDefaultAnalyzerModel('xai', result)).toBe('grok-4-0709'); + }); + + it('filters out embedding models — falls back to preset default', () => { + const result = makeRunResult('openai', 'text-embedding-3-large'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); + }); + + it('filters out tts models — falls back to preset default', () => { + const result = makeRunResult('openai', 'tts-1-hd'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); + }); + + it('does not filter vision/text models with "image" in the name', () => { + const result = makeRunResult('openai', 'gpt-5-image-understanding'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('gpt-5-image-understanding'); + }); + it('returns preset default when providers differ', () => { const result = makeRunResult('ollama', 'qwen3:30b'); expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); @@ -206,6 +226,62 @@ describe('calculateKSM edge cases', () => { }); }); +// ============================================================================= +// extractJson — brace-matching JSON scanner +// ============================================================================= + +describe('extractJson', () => { + it('passes through clean JSON unchanged', () => { + const json = '{"key": "value", "num": 42}'; + expect(extractJson(json)).toBe(json); + }); + + it('strips leading explanation text', () => { + const input = 'Here is my analysis of the run:\n\n{"score": 75}'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('strips trailing explanation text', () => { + const input = '{"score": 75}\n\nI hope this helps with your evaluation.'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('strips markdown fences', () => { + const input = '```json\n{"score": 75}\n```'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('handles fences not at start of string', () => { + const input = 'Here\'s the result:\n```json\n{"score": 75}\n```'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('handles nested objects', () => { + const json = '{"outer": {"inner": {"deep": 1}}, "arr": [{}]}'; + expect(extractJson(`Some preamble\n${json}\nSome postamble`)).toBe(json); + }); + + it('handles braces inside string values', () => { + const json = '{"key": "value with { braces } inside"}'; + expect(extractJson(json)).toBe(json); + }); + + it('returns from first brace to end for unclosed JSON', () => { + const input = '{"key": "value", "nested": {"open": true'; + expect(extractJson(input)).toBe(input); + }); + + it('extracts first JSON object when multiple are present', () => { + const input = '{"first": 1}\n{"second": 2}'; + expect(extractJson(input)).toBe('{"first": 1}'); + }); + + it('returns original text when no braces found', () => { + const input = 'no json here at all'; + expect(extractJson(input)).toBe(input); + }); +}); + // ============================================================================= // parseAnalysisResponse — malformed LLM output handling // =============================================================================