From 972fec4817cb45420e4018f51a1c29390f4f6f1c Mon Sep 17 00:00:00 2001 From: aarti jakhar Date: Fri, 27 Feb 2026 21:07:32 +0530 Subject: [PATCH 1/4] fix: skip export when analysis missing, fix dead report URL - Skip export prompt entirely when no analysis available (API quota/rate limit), guide user to run `oasis analyze` then `oasis report` instead - Reorder export menu so "Done" is default after first action - Replace dead oasis.kryptsec.com URL with GitHub repo in reports --- src/lib/export.ts | 22 ++++++++++++++++------ src/lib/report.ts | 4 ++-- tests/unit/report.test.ts | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/lib/export.ts b/src/lib/export.ts index fd48500..dc636a6 100644 --- a/src/lib/export.ts +++ b/src/lib/export.ts @@ -14,21 +14,31 @@ export async function promptExport( ): Promise { console.log(); if (!analysis) { - console.log(colors.gray(' No analysis available — export will not include scores.')); + console.log(colors.gray(' No analysis available — skipping export.')); + console.log(colors.gray(` Run analysis first: oasis analyze ${result.id}`)); + console.log(colors.gray(` Then export via: oasis report ${result.id}`)); + return; } console.log(colors.gray(` More export formats available via: oasis report ${result.id}`)); + let hasActed = false; while (true) { + const exportChoices = [ + { name: 'Copy share card to clipboard', value: 'share' as const }, + { name: 'Save HTML report', value: 'html' as const }, + ]; + const doneChoice = { name: hasActed ? 'Done' : colors.gray('Done'), value: 'done' as const }; + const choices = hasActed + ? [doneChoice, ...exportChoices] + : [...exportChoices, doneChoice]; + const action = await select({ message: 'Share results?', - choices: [ - { name: 'Copy share card to clipboard', value: 'share' as const }, - { name: 'Save HTML report', value: 'html' as const }, - { name: colors.gray('Done'), value: 'done' as const }, - ], + choices, }); if (action === 'done') break; + hasActed = true; if (action === 'share') { const card = generateShareCard(result, analysis, ksmScore); diff --git a/src/lib/report.ts b/src/lib/report.ts index 94ab24d..69a729e 100644 --- a/src/lib/report.ts +++ b/src/lib/report.ts @@ -554,7 +554,7 @@ export function generateShareCard( md += `> **Approach:** ${analysis.behavior.approach} \u2014 ${analysis.narrative.summary.split('.')[0]}.\n\n`; } - md += `> Benchmarked with [OASIS](https://oasis.kryptsec.com)\n`; + md += `> Benchmarked with [OASIS](https://github.com/KryptSec/oasis)\n`; return md; } @@ -749,7 +749,7 @@ export function generateHtmlReport( ${analysisHtml}
- Generated by OASIS \u2014 AI Security Benchmarking + Generated by OASIS \u2014 AI Security Benchmarking
`; diff --git a/tests/unit/report.test.ts b/tests/unit/report.test.ts index ec8c52d..5f1c35f 100644 --- a/tests/unit/report.test.ts +++ b/tests/unit/report.test.ts @@ -357,7 +357,7 @@ describe('generateShareCard', () => { it('ends with OASIS link', () => { const card = generateShareCard(successfulRun); - expect(card).toContain('oasis.kryptsec.com'); + expect(card).toContain('github.com/KryptSec/oasis'); }); }); From 7a15f066c448b68cd0a717373db9dcda7294f6a7 Mon Sep 17 00:00:00 2001 From: aarti jakhar Date: Sat, 28 Feb 2026 04:22:13 +0530 Subject: [PATCH 2/4] fix: analysis parsing failures and same-provider model resolution - Add robust JSON extraction (brace-matching) for LLM responses wrapped in explanation text or trailing content - Increase MAX_COMPLETION_TOKENS from 4096 to 8192 to prevent truncation - Use benchmark model for analysis instead of hardcoding claude-sonnet, with blocklist for non-text models (image/embed/tts) - Display actual analyzer model in results instead of hardcoded default - Prevent wrong API key passthrough when analyzer provider differs --- src/commands/run.ts | 2 +- src/lib/analyzer.ts | 47 +++++++++++++++++++++++++++---------- src/lib/constants.ts | 2 +- tests/unit/analyzer.test.ts | 15 ++++++++++++ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/commands/run.ts b/src/commands/run.ts index 6816d3e..c4808ad 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -245,7 +245,7 @@ export const runCommand = new Command('run') ? normalizeProvider(options.analyzerProvider || provider) : ''; const analyzerApiKey = analyze - ? (options.analyzerKey || getApiKey(analyzerProvider) || resolvedApiKey) + ? (options.analyzerKey || getApiKey(analyzerProvider) || (normalizeProvider(provider) === analyzerProvider ? resolvedApiKey : undefined)) : undefined; const analyzerBaseUrl = analyze ? (options.analyzerUrl || getEffectiveProviderUrl(analyzerProvider) || undefined) diff --git a/src/lib/analyzer.ts b/src/lib/analyzer.ts index b80e32c..4f2ae4b 100644 --- a/src/lib/analyzer.ts +++ b/src/lib/analyzer.ts @@ -232,18 +232,39 @@ Return ONLY the JSON object, no other text.`; // Response Parser // ============================================================================= +function extractJson(text: string): string { + let s = text.trim(); + + // Strip markdown fences anywhere in the response + s = s.replace(/^```json\s*/i, '').replace(/^```\s*/, '').replace(/\s*```\s*$/, '').trim(); + + // Find the outermost { ... } — always run the brace scanner to strip trailing text + const start = s.startsWith('{') ? 0 : s.indexOf('{'); + if (start === -1) return s; + let depth = 0; + let inString = false; + let escape = false; + for (let i = start; i < s.length; i++) { + const ch = s[i]; + if (escape) { escape = false; continue; } + if (ch === '\\' && inString) { escape = true; continue; } + if (ch === '"' && !escape) { inString = !inString; continue; } + if (inString) continue; + if (ch === '{') depth++; + else if (ch === '}') { depth--; if (depth === 0) return s.substring(start, i + 1); } + } + // Unclosed — return from first '{' to end (let JSON.parse report the real error) + return s.substring(start); +} + export async function parseAnalysisResponse( response: string, runId: string, result: RunResult, - challengeConfig?: ChallengeConfig + challengeConfig?: ChallengeConfig, + analyzerModel?: string, ): Promise { - let jsonStr = response.trim(); - - if (jsonStr.startsWith('```json')) jsonStr = jsonStr.slice(7); - else if (jsonStr.startsWith('```')) jsonStr = jsonStr.slice(3); - if (jsonStr.endsWith('```')) jsonStr = jsonStr.slice(0, -3); - jsonStr = jsonStr.trim(); + const jsonStr = extractJson(response); try { const parsed = AnalysisResponseSchema.parse(JSON.parse(jsonStr)); @@ -251,7 +272,7 @@ export async function parseAnalysisResponse( const analysisResult: AnalysisResult = { runId, analyzedAt: new Date(), - analyzerModel: DEFAULT_ANALYZER_MODEL, + analyzerModel: analyzerModel || DEFAULT_ANALYZER_MODEL, attackChain: { phases: parsed.attackChain.phases, techniques: parsed.attackChain.techniques, @@ -342,7 +363,7 @@ export async function parseAnalysisResponse( return { runId, analyzedAt: new Date(), - analyzerModel: DEFAULT_ANALYZER_MODEL, + analyzerModel: analyzerModel || DEFAULT_ANALYZER_MODEL, parseFailed: true, attackChain: { phases: [], techniques: [], killChainCoverage: [] }, narrative: { summary: 'Analysis parsing failed', detailed: `Error: ${error}`, keyFindings: [] }, @@ -447,10 +468,12 @@ export function resolveDefaultAnalyzerModel(analyzerProvider: string, benchmarkR const preset = resolveProvider(analyzerProvider); - // Same provider as benchmark — use the benchmark model since we know it's available + // Same provider as benchmark — use the benchmark model (user's choice), but + // filter out known non-text models that can't do chat completions. const benchmarkProvider = normalizeProvider(benchmarkResult.model); if (benchmarkProvider === normalizeProvider(analyzerProvider)) { - return benchmarkResult.modelVersion || preset?.models[0] || DEFAULT_ANALYZER_MODEL; + const isTextModel = !/imagine|image|embed|tts|whisper|dall-e/i.test(benchmarkResult.modelVersion || ''); + return (isTextModel ? benchmarkResult.modelVersion : null) || preset?.models[0] || DEFAULT_ANALYZER_MODEL; } // Different provider — try preset default, fall back to Claude @@ -543,7 +566,7 @@ export async function analyzeRun( responseText = await callOpenAIAnalyzer(apiKey, baseUrl, analyzerModel, prompt); } - return parseAnalysisResponse(responseText, result.id, result, options.challengeConfig); + return parseAnalysisResponse(responseText, result.id, result, options.challengeConfig, analyzerModel); } export async function analyzeExistingRun( diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 0c8f7a9..c57f628 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -1,7 +1,7 @@ // Named constants — replacing magic numbers across the codebase // API limits -export const MAX_COMPLETION_TOKENS = 4096; +export const MAX_COMPLETION_TOKENS = 8192; // Output truncation export const STEP_OUTPUT_LIMIT = 10_000; // Stored in step records diff --git a/tests/unit/analyzer.test.ts b/tests/unit/analyzer.test.ts index 41a0bd5..af7276e 100644 --- a/tests/unit/analyzer.test.ts +++ b/tests/unit/analyzer.test.ts @@ -144,6 +144,21 @@ describe('resolveDefaultAnalyzerModel', () => { expect(resolveDefaultAnalyzerModel('xai', result)).toBe('grok-3-latest'); }); + it('filters out image models — falls back to preset default', () => { + const result = makeRunResult('xai', 'grok-imagine-image'); + expect(resolveDefaultAnalyzerModel('xai', result)).toBe('grok-4-0709'); + }); + + it('filters out embedding models — falls back to preset default', () => { + const result = makeRunResult('openai', 'text-embedding-3-large'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); + }); + + it('filters out tts models — falls back to preset default', () => { + const result = makeRunResult('openai', 'tts-1-hd'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); + }); + it('returns preset default when providers differ', () => { const result = makeRunResult('ollama', 'qwen3:30b'); expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); From 1ff1ae15d514521364e9393ee1ff2250d030478d Mon Sep 17 00:00:00 2001 From: aarti jakhar Date: Mon, 2 Mar 2026 22:21:27 +0530 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20address=20PR=20#61=20review=20?= =?UTF-8?q?=E2=80=94=20extractJson=20tests,=20anchored=20regex,=20revert?= =?UTF-8?q?=20token=20bump?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Export extractJson and add 10 direct unit tests covering: clean passthrough, leading/trailing text, markdown fences, nested objects, braces in strings, unclosed JSON, multiple objects, no-JSON input - Replace loose /imagine|image|embed|.../ regex with anchored NON_TEXT_MODEL_PATTERNS blocklist to prevent false positives on vision models like gpt-5-image-understanding - Revert MAX_COMPLETION_TOKENS to 4096 — extractJson already strips preamble text that caused truncation, token bump is redundant --- src/lib/analyzer.ts | 13 ++++++-- src/lib/constants.ts | 2 +- tests/unit/analyzer.test.ts | 63 ++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/lib/analyzer.ts b/src/lib/analyzer.ts index 4f2ae4b..47fb2b4 100644 --- a/src/lib/analyzer.ts +++ b/src/lib/analyzer.ts @@ -232,7 +232,7 @@ Return ONLY the JSON object, no other text.`; // Response Parser // ============================================================================= -function extractJson(text: string): string { +export function extractJson(text: string): string { let s = text.trim(); // Strip markdown fences anywhere in the response @@ -472,7 +472,16 @@ export function resolveDefaultAnalyzerModel(analyzerProvider: string, benchmarkR // filter out known non-text models that can't do chat completions. const benchmarkProvider = normalizeProvider(benchmarkResult.model); if (benchmarkProvider === normalizeProvider(analyzerProvider)) { - const isTextModel = !/imagine|image|embed|tts|whisper|dall-e/i.test(benchmarkResult.modelVersion || ''); + const NON_TEXT_MODEL_PATTERNS: RegExp[] = [ + /^grok-imagine-/i, + /^dall-e-/i, + /^tts-/i, + /^whisper-/i, + /[-.]embed(ding)?s?[-.:]/i, + /^text-embed(ding)?/i, + ]; + const modelName = benchmarkResult.modelVersion || ''; + const isTextModel = !NON_TEXT_MODEL_PATTERNS.some(p => p.test(modelName)); return (isTextModel ? benchmarkResult.modelVersion : null) || preset?.models[0] || DEFAULT_ANALYZER_MODEL; } diff --git a/src/lib/constants.ts b/src/lib/constants.ts index c57f628..0c8f7a9 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -1,7 +1,7 @@ // Named constants — replacing magic numbers across the codebase // API limits -export const MAX_COMPLETION_TOKENS = 8192; +export const MAX_COMPLETION_TOKENS = 4096; // Output truncation export const STEP_OUTPUT_LIMIT = 10_000; // Stored in step records diff --git a/tests/unit/analyzer.test.ts b/tests/unit/analyzer.test.ts index af7276e..578a4e7 100644 --- a/tests/unit/analyzer.test.ts +++ b/tests/unit/analyzer.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { calculateKSM, fallbackOverallScore } from '../../src/lib/scoring.js'; -import { resolveDefaultAnalyzerModel, DEFAULT_ANALYZER_MODEL, parseAnalysisResponse } from '../../src/lib/analyzer.js'; +import { resolveDefaultAnalyzerModel, DEFAULT_ANALYZER_MODEL, parseAnalysisResponse, extractJson } from '../../src/lib/analyzer.js'; import type { RunResult } from '../../src/lib/types.js'; function makeRunResult(model: string, modelVersion: string): RunResult { @@ -159,6 +159,11 @@ describe('resolveDefaultAnalyzerModel', () => { expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); }); + it('does not filter vision/text models with "image" in the name', () => { + const result = makeRunResult('openai', 'gpt-5-image-understanding'); + expect(resolveDefaultAnalyzerModel('openai', result)).toBe('gpt-5-image-understanding'); + }); + it('returns preset default when providers differ', () => { const result = makeRunResult('ollama', 'qwen3:30b'); expect(resolveDefaultAnalyzerModel('openai', result)).toBe('o3'); @@ -221,6 +226,62 @@ describe('calculateKSM edge cases', () => { }); }); +// ============================================================================= +// extractJson — brace-matching JSON scanner +// ============================================================================= + +describe('extractJson', () => { + it('passes through clean JSON unchanged', () => { + const json = '{"key": "value", "num": 42}'; + expect(extractJson(json)).toBe(json); + }); + + it('strips leading explanation text', () => { + const input = 'Here is my analysis of the run:\n\n{"score": 75}'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('strips trailing explanation text', () => { + const input = '{"score": 75}\n\nI hope this helps with your evaluation.'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('strips markdown fences', () => { + const input = '```json\n{"score": 75}\n```'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('handles fences not at start of string', () => { + const input = 'Here\'s the result:\n```json\n{"score": 75}\n```'; + expect(extractJson(input)).toBe('{"score": 75}'); + }); + + it('handles nested objects', () => { + const json = '{"outer": {"inner": {"deep": 1}}, "arr": [{}]}'; + expect(extractJson(`Some preamble\n${json}\nSome postamble`)).toBe(json); + }); + + it('handles braces inside string values', () => { + const json = '{"key": "value with { braces } inside"}'; + expect(extractJson(json)).toBe(json); + }); + + it('returns from first brace to end for unclosed JSON', () => { + const input = '{"key": "value", "nested": {"open": true'; + expect(extractJson(input)).toBe(input); + }); + + it('extracts first JSON object when multiple are present', () => { + const input = '{"first": 1}\n{"second": 2}'; + expect(extractJson(input)).toBe('{"first": 1}'); + }); + + it('returns original text when no braces found', () => { + const input = 'no json here at all'; + expect(extractJson(input)).toBe(input); + }); +}); + // ============================================================================= // parseAnalysisResponse — malformed LLM output handling // ============================================================================= From e944b85fc286fa62efcc1d31c999502a778b4d13 Mon Sep 17 00:00:00 2001 From: Marshall Livingston Date: Mon, 2 Mar 2026 11:53:15 -0700 Subject: [PATCH 4/4] =?UTF-8?q?Revert=20oasis.kryptsec.com=20URL=20replace?= =?UTF-8?q?ment=20=E2=80=94=20it's=20not=20dead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original PR swapped oasis.kryptsec.com for the GitHub repo URL in share cards and HTML reports. That domain is live. Put it back. --- src/lib/report.ts | 4 ++-- tests/unit/report.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/report.ts b/src/lib/report.ts index 69a729e..94ab24d 100644 --- a/src/lib/report.ts +++ b/src/lib/report.ts @@ -554,7 +554,7 @@ export function generateShareCard( md += `> **Approach:** ${analysis.behavior.approach} \u2014 ${analysis.narrative.summary.split('.')[0]}.\n\n`; } - md += `> Benchmarked with [OASIS](https://github.com/KryptSec/oasis)\n`; + md += `> Benchmarked with [OASIS](https://oasis.kryptsec.com)\n`; return md; } @@ -749,7 +749,7 @@ export function generateHtmlReport( ${analysisHtml}
- Generated by OASIS \u2014 AI Security Benchmarking + Generated by OASIS \u2014 AI Security Benchmarking
`; diff --git a/tests/unit/report.test.ts b/tests/unit/report.test.ts index 5f1c35f..ec8c52d 100644 --- a/tests/unit/report.test.ts +++ b/tests/unit/report.test.ts @@ -357,7 +357,7 @@ describe('generateShareCard', () => { it('ends with OASIS link', () => { const card = generateShareCard(successfulRun); - expect(card).toContain('github.com/KryptSec/oasis'); + expect(card).toContain('oasis.kryptsec.com'); }); });