From 91c402949cf9c7764c02c8b2749b9e6ea077e58f Mon Sep 17 00:00:00 2001 From: styltsou Date: Sun, 28 Jun 2026 14:00:34 +0300 Subject: [PATCH] feat: add x-toolsafe-ignore inline suppression --- README.md | 27 ++++- docs/RULES.md | 41 +++++++ package.json | 2 +- src/core/analyze.ts | 2 + src/core/suppression.ts | 56 ++++++++++ tests/core/suppression.test.ts | 197 +++++++++++++++++++++++++++++++++ 6 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 src/core/suppression.ts create mode 100644 tests/core/suppression.test.ts diff --git a/README.md b/README.md index 86f7c21..60f9ff4 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,31 @@ Rules are grouped into categories. The **Agent-specific?** column tells you whet Run `toolsafe rules` to see the full list with current severities. +## Inline Suppression + +Operations can suppress specific rules or all rules using vendor extensions. This lets you adopt ToolSafe incrementally — mark known findings as accepted so CI passes while the remaining findings get addressed. + +**Suppress specific rules on an operation:** + +```yaml +/users/{id}: + delete: + operationId: deleteUser + x-toolsafe-ignore: + - safety/destructive-requires-guard +``` + +**Suppress all rules on an operation:** + +```yaml +/users/{id}: + delete: + operationId: deleteUser + x-toolsafe-ignore-all: true +``` + +Both extensions apply at the operation level only. Suppressed findings never appear in output, regardless of their severity. + ## How rules match operations Rules use a tokenized intent-matching helper (`getOperationIntentText` / `hasOperationIntentKeyword`) that scopes keyword matching to operation IDs, names, methods, paths, summaries, and tags — deliberately excluding free-text `description` prose, which is where naive substring matching tends to produce false positives (e.g. a read-only operation whose description happens to mention "cancel" in passing). See `docs/RULES.md` for the matching approach per rule and known precision tradeoffs. @@ -304,7 +329,7 @@ This is a heuristic, explainable approach, not a full semantic parse of API inte ToolSafe is early (v0.x) and the rule engine is intentionally simple right now. Rough priority order, subject to change: - **Selector-based rule matching.** Rules currently match against a tokenized text blob per operation. The plan is to move toward JSONPath-style scoped selectors per rule (closer to how Spectral/vacuum target OpenAPI documents) so a rule only ever sees the exact field it's meant to check, rather than matching across operation IDs, paths, and summaries indiscriminately. This should remove a category of false positive at the source instead of patching it rule-by-rule. -- **Inline suppression.** A way to mark a specific operation as reviewed and intentionally accepted (e.g. an `x-toolsafe-ignore: rule-id` vendor extension), so teams can adopt ToolSafe incrementally instead of fixing every finding before CI goes green. +- **Inline suppression.** ✅ Implemented. A way to mark a specific operation as reviewed and intentionally accepted (e.g. an `x-toolsafe-ignore: rule-id` vendor extension), so teams can adopt ToolSafe incrementally instead of fixing every finding before CI goes green. - **Precision validation against real-world specs.** Running and publishing results against large public OpenAPI specs (Stripe, GitHub, etc.), not just synthetic fixtures, with measured false-positive rates per rule. - **More complete `$ref` resolution.** Rules should see the fully dereferenced schema graph rather than relying on each rule to handle references defensively. - **Guard policy / eval generation maturity.** `toolsafe generate` currently produces advisory drafts; the goal is for generated guard policies and eval cases to be closer to drop-in usable rather than a starting point that needs heavy editing. diff --git a/docs/RULES.md b/docs/RULES.md index e84a4e0..61861f4 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -44,6 +44,47 @@ Each finding includes: Findings are sorted by severity, path, method, and rule ID. This keeps reports stable and makes snapshot tests useful. +## Inline Suppression + +Operations can suppress specific rules or all rules using vendor extensions in the OpenAPI spec. This allows teams to adopt ToolSafe incrementally — mark known/accepted findings so CI can go green while remaining findings are addressed. + +### Suppress specific rules + +Add `x-toolsafe-ignore` as a vendor extension on an operation with a list of rule IDs to suppress: + +```yaml +/users/{id}: + delete: + operationId: deleteUser + x-toolsafe-ignore: + - safety/destructive-requires-guard + responses: + '204': + description: Deleted +``` + +Only findings whose rule ID appears in the list are suppressed for that operation. + +### Suppress all rules on an operation + +Add `x-toolsafe-ignore-all: true` to suppress every finding for that operation: + +```yaml +/users/{id}: + delete: + operationId: deleteUser + x-toolsafe-ignore-all: true + responses: + '204': + description: Deleted +``` + +### Scope + +Both extensions apply at the operation level only. They are not inherited from the root or path-level OpenAPI objects. Findings for operations without these extensions are unaffected. + +Suppression is applied after rule execution and after severity overrides from `toolsafe.config.json`, so a suppressed finding never appears in the output regardless of its configured severity. + ## Adding Or Changing Rules Rule implementations live under `src/rules/`. The default registry is `src/rules/index.ts`. diff --git a/package.json b/package.json index 8691708..c562ce6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "toolsafe", - "version": "0.1.2", + "version": "0.2.0", "description": "Lint OpenAPI specs for AI agent safety — detect risky operations, missing guards, destructive endpoints, and schema issues before exposing APIs to LLMs and agents", "keywords": [ "agent", diff --git a/src/core/analyze.ts b/src/core/analyze.ts index bcff012..a9e8bf5 100644 --- a/src/core/analyze.ts +++ b/src/core/analyze.ts @@ -7,6 +7,7 @@ import { normalizeOpenApi } from '@/core/normalize'; import { classifyToolRisks } from '@/core/risk'; import { calculateScores } from '@/core/scoring'; import type { AnalysisResult, Finding, NormalizedTool, ToolRiskSummary } from '@/core/types'; +import { suppressIgnoredFindings } from '@/core/suppression'; /** * Runs ToolSafe's complete deterministic analysis pipeline for one local file. @@ -25,6 +26,7 @@ export async function analyzeOpenApi( const activeRules = filterRules(defaultRules, config); let findings = runRules(tools, activeRules); findings = overrideSeverities(findings, config); + findings = suppressIgnoredFindings(findings, tools); const toolRisks = classifyToolRisks(tools); return { diff --git a/src/core/suppression.ts b/src/core/suppression.ts new file mode 100644 index 0000000..8efef02 --- /dev/null +++ b/src/core/suppression.ts @@ -0,0 +1,56 @@ +import type { Finding, NormalizedTool } from '@/core/types'; +import { isObject } from '@/core/objects'; + +const IGNORE_EXTENSION = 'x-toolsafe-ignore'; +const IGNORE_ALL_EXTENSION = 'x-toolsafe-ignore-all'; + +export function getIgnoredRuleIds(tool: NormalizedTool): string[] { + if (!isObject(tool.rawOperation)) { + return []; + } + + const value = (tool.rawOperation as Record)[IGNORE_EXTENSION]; + + if (!Array.isArray(value)) { + return []; + } + + return value.filter((id): id is string => typeof id === 'string'); +} + +export function isAllIgnored(tool: NormalizedTool): boolean { + if (!isObject(tool.rawOperation)) { + return false; + } + + return (tool.rawOperation as Record)[IGNORE_ALL_EXTENSION] === true; +} + +export function suppressIgnoredFindings(findings: Finding[], tools: NormalizedTool[]): Finding[] { + const allIgnored = new Set(); + const specificIgnored = new Map>(); + + for (const tool of tools) { + if (isAllIgnored(tool)) { + allIgnored.add(tool.id); + } else { + const ids = getIgnoredRuleIds(tool); + if (ids.length > 0) { + specificIgnored.set(tool.id, new Set(ids)); + } + } + } + + if (allIgnored.size === 0 && specificIgnored.size === 0) { + return findings; + } + + return findings.filter((finding) => { + if (allIgnored.has(finding.toolId)) { + return false; + } + + const specific = specificIgnored.get(finding.toolId); + return specific ? !specific.has(finding.ruleId) : true; + }); +} diff --git a/tests/core/suppression.test.ts b/tests/core/suppression.test.ts new file mode 100644 index 0000000..de45eb0 --- /dev/null +++ b/tests/core/suppression.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, test } from 'bun:test'; +import type { Finding, NormalizedTool } from '@/core/types'; +import { getIgnoredRuleIds, isAllIgnored, suppressIgnoredFindings } from '@/core/suppression'; + +const baseTool: NormalizedTool = { + id: 'testTool', + name: 'testTool', + method: 'GET', + path: '/test', + tags: [], + parameters: [], + responses: [], + rawOperation: {}, +}; + +describe('getIgnoredRuleIds', () => { + test('returns empty array when no x-toolsafe-ignore extension exists', () => { + expect(getIgnoredRuleIds(baseTool)).toEqual([]); + }); + + test('returns empty array when rawOperation is empty', () => { + expect(getIgnoredRuleIds({ ...baseTool, rawOperation: {} })).toEqual([]); + }); + + test('returns empty array for non-array value', () => { + const tool = { ...baseTool, rawOperation: { 'x-toolsafe-ignore': 'not-an-array' } }; + expect(getIgnoredRuleIds(tool)).toEqual([]); + }); + + test('returns list of rule IDs from x-toolsafe-ignore', () => { + const tool = { + ...baseTool, + rawOperation: { + 'x-toolsafe-ignore': ['safety/destructive-requires-guard', 'errors/missing-error-schema'], + }, + }; + expect(getIgnoredRuleIds(tool)).toEqual([ + 'safety/destructive-requires-guard', + 'errors/missing-error-schema', + ]); + }); + + test('filters out non-string entries from the array', () => { + const tool = { + ...baseTool, + rawOperation: { + 'x-toolsafe-ignore': ['safety/destructive-requires-guard', 123, true], + }, + }; + expect(getIgnoredRuleIds(tool)).toEqual(['safety/destructive-requires-guard']); + }); + + test('returns empty array when rawOperation is null', () => { + const tool = { ...baseTool, rawOperation: null } as unknown as NormalizedTool; + expect(getIgnoredRuleIds(tool)).toEqual([]); + }); +}); + +describe('isAllIgnored', () => { + test('returns false when no x-toolsafe-ignore-all extension exists', () => { + expect(isAllIgnored(baseTool)).toBe(false); + }); + + test('returns true when x-toolsafe-ignore-all is true', () => { + const tool = { ...baseTool, rawOperation: { 'x-toolsafe-ignore-all': true } }; + expect(isAllIgnored(tool)).toBe(true); + }); + + test('returns false when x-toolsafe-ignore-all is false', () => { + const tool = { ...baseTool, rawOperation: { 'x-toolsafe-ignore-all': false } }; + expect(isAllIgnored(tool)).toBe(false); + }); + + test('returns false when x-toolsafe-ignore-all is a non-boolean truthy value', () => { + const tool = { ...baseTool, rawOperation: { 'x-toolsafe-ignore-all': 'yes' } }; + expect(isAllIgnored(tool)).toBe(false); + }); + + test('returns false when rawOperation is null', () => { + const tool = { ...baseTool, rawOperation: null } as unknown as NormalizedTool; + expect(isAllIgnored(tool)).toBe(false); + }); +}); + +function makeFinding(overrides: Partial): Finding { + return { + ruleId: 'safety/destructive-requires-guard', + severity: 'error', + category: 'safety', + toolId: 'deleteUser', + toolName: 'deleteUser', + method: 'DELETE', + path: '/users/{id}', + message: 'test', + recommendation: 'test', + ...overrides, + }; +} + +function makeTool(overrides: Partial): NormalizedTool { + return { + id: 'testTool', + name: 'testTool', + method: 'GET', + path: '/test', + tags: [], + parameters: [], + responses: [], + rawOperation: {}, + ...overrides, + }; +} + +describe('suppressIgnoredFindings', () => { + test('returns all findings when no tools have suppression extensions', () => { + const findings = [makeFinding({ toolId: 'op1' }), makeFinding({ toolId: 'op2' })]; + const tools = [makeTool({ id: 'op1' }), makeTool({ id: 'op2' })]; + + expect(suppressIgnoredFindings(findings, tools)).toEqual(findings); + }); + + test('suppresses findings for a specific rule ID via x-toolsafe-ignore', () => { + const f1 = makeFinding({ toolId: 'op1', ruleId: 'safety/destructive-requires-guard' }); + const f2 = makeFinding({ toolId: 'op1', ruleId: 'errors/missing-error-schema' }); + const tools = [ + makeTool({ + id: 'op1', + rawOperation: { 'x-toolsafe-ignore': ['safety/destructive-requires-guard'] }, + }), + ]; + + const result = suppressIgnoredFindings([f1, f2], tools); + + expect(result).toEqual([f2]); + }); + + test('suppresses all findings for an operation via x-toolsafe-ignore-all', () => { + const f1 = makeFinding({ toolId: 'op1', ruleId: 'safety/destructive-requires-guard' }); + const f2 = makeFinding({ toolId: 'op1', ruleId: 'errors/missing-error-schema' }); + const tools = [ + makeTool({ + id: 'op1', + rawOperation: { 'x-toolsafe-ignore-all': true }, + }), + ]; + + const result = suppressIgnoredFindings([f1, f2], tools); + + expect(result).toEqual([]); + }); + + test('does not suppress findings for unrelated operations', () => { + const f1 = makeFinding({ toolId: 'op1', ruleId: 'safety/destructive-requires-guard' }); + const f2 = makeFinding({ toolId: 'op2', ruleId: 'errors/missing-error-schema' }); + const tools = [ + makeTool({ id: 'op1', rawOperation: { 'x-toolsafe-ignore-all': true } }), + makeTool({ id: 'op2' }), + ]; + + const result = suppressIgnoredFindings([f1, f2], tools); + + expect(result).toEqual([f2]); + }); + + test('returns empty array when all findings are suppressed', () => { + const findings = [makeFinding({ toolId: 'op1' })]; + const tools = [makeTool({ id: 'op1', rawOperation: { 'x-toolsafe-ignore-all': true } })]; + + expect(suppressIgnoredFindings(findings, tools)).toEqual([]); + }); + + test('handles empty findings', () => { + const tools = [makeTool({ id: 'op1', rawOperation: { 'x-toolsafe-ignore-all': true } })]; + + expect(suppressIgnoredFindings([], tools)).toEqual([]); + }); + + test('handles empty tools', () => { + const findings = [makeFinding({ toolId: 'op1' })]; + + expect(suppressIgnoredFindings(findings, [])).toEqual(findings); + }); + + test('ignores x-toolsafe-ignore with an invalid rule ID that never matches', () => { + const f1 = makeFinding({ toolId: 'op1', ruleId: 'safety/destructive-requires-guard' }); + const tools = [ + makeTool({ + id: 'op1', + rawOperation: { 'x-toolsafe-ignore': ['nonexistent/rule'] }, + }), + ]; + + const result = suppressIgnoredFindings([f1], tools); + + expect(result).toEqual([f1]); + }); +});