diff --git a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json index bfeb6cd23..5595a0c5e 100644 --- a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json +++ b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.60", + "version": "0.1.83", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index 38f0355be..d85b95a22 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -44,6 +44,18 @@ "directory_snapshot": {"snapshotDigest"}, } DISPOSITIONS = {"reported", "no_issue_found", "rejected", "not_applicable", "needs_follow_up"} +NON_COVERAGE_TARGET_WARNINGS = { + "Directory contents changed while the scan was running; " + "results were saved for the original snapshot.", + "The scanned Git repository became unavailable while the scan was running; " + "results were saved for the original revision.", + "Repository HEAD changed while the scan was running; " + "results were saved for the original revision.", + "Working-tree contents changed while the scan was running; " + "results were saved for the original snapshot.", + "The scan target became unavailable while the scan was running; " + "results were saved for the original revision or snapshot.", +} SARIF_LEVELS = { "critical": "error", "high": "error", @@ -1023,6 +1035,25 @@ def _recover_unsealed_coverage( partial = True if partial: coverage["completeness"] = "partial" + elif ( + completeness == "partial" + and coverage.get("mode") == "deep_repository" + and coverage["surfaces"] + and not coverage["deferred"] + and all( + warning in NON_COVERAGE_TARGET_WARNINGS + or re.fullmatch( + r"Recovered finding [0-9]+: " + r"(?:normalized [a-z, ]+|retained stronger duplicate logical finding)\.", + warning, + ) + for warning in warnings + ) + ): + coverage["completeness"] = "complete" + warnings.append( + "Recovered Deep Scan coverage marked partial without deferred review work." + ) def _recover_unsealed_hardening( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py index 7d4e1c979..785e94d65 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py @@ -47,6 +47,9 @@ _PUBLICATION_FOLLOW_UP_WARNING = ( "Saved scan evidence remains on disk; result publication needs follow-up:" ) +_UNVERIFIED_COVERAGE_WARNING = ( + "Saved scan source is incomplete or has unverified coverage; coverage remains partial." +) @dataclass(frozen=True) @@ -252,6 +255,11 @@ def merge_saved_results( } if not isinstance(parent["findings"], list): raise ContractError("Saved parent draft has no findings array") + if ( + parent_scan.get("complete", True) is not True + and _UNVERIFIED_COVERAGE_WARNING not in warnings + ): + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if not parent_scan.get("sealedAt"): payload = _encoded(parent) write_scan_local_bytes( @@ -277,6 +285,7 @@ def merge_saved_results( source_digests.update(parent_preserved_sources) paths: dict[str, str | None] = {} current_results: set[str] = set() + required_results: set[str] = set() reducer_outputs: list[tuple[Any, str, list[str], int]] = [] accepted_reducers = [ worker @@ -293,6 +302,7 @@ def merge_saved_results( try: latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[latest_reducer] = None + required_results.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") @@ -351,6 +361,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[current_path] = worker["id"] current_results.add(current_path) + if worker["status"] == "succeeded": + required_results.add(current_path) except ValueError: warnings.append("Skipped a worker result outside the scan directory.") @@ -381,6 +393,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}") + elif relative in required_results and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if frozen_source_digests is not None: if frozen_source_digests.keys() - source_digests.keys(): raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") @@ -577,16 +591,30 @@ def valid_finding(value: Any) -> bool: for saved_path, current, saved_worker in sources ) ) - if ( - (relative != "parent" or not parent_manifest) - and not superseded - and ( - draft.get("complete") is False - or draft["coverage"].get("completeness") != "complete" - ) - and coverage.get("completeness") in {"complete", "unknown"} - ): - coverage["completeness"] = "partial" + if (relative != "parent" or not parent_manifest) and not superseded: + source_coverage = draft["coverage"] + source_completeness = source_coverage.get("completeness") + source_complete = draft.get("complete", True) is True + if ( + not source_complete + or ( + source_completeness != "complete" + and ( + worker_id is not None + or source_completeness != "partial" + or coverage.get("completeness") == "unknown" + ) + ) + or any( + not isinstance(source_coverage.get(field), list) + for field in ("surfaces", "explicitExclusions", "deferred") + ) + ) and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) + if ( + not source_complete or source_completeness != "complete" + ) and coverage.get("completeness") in {"complete", "unknown"}: + coverage["completeness"] = "partial" if superseded and not stopped: continue if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict): @@ -813,7 +841,11 @@ def valid_finding(value: Any) -> bool: used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) - if stopped or any(warning not in initial_warnings for warning in warnings): + if ( + stopped + or _UNVERIFIED_COVERAGE_WARNING in warnings + or any(warning not in initial_warnings for warning in warnings) + ): coverage["completeness"] = "partial" if stopped: if not isinstance(coverage.get("deferred"), list): diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index 95861c52e..7a5348aca 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -9,7 +9,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.60" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.83" as const; const PACKAGE_NAME = "@openai/codex-security"; diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 02c66b8d6..f4b741a26 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -66,31 +66,34 @@ interface OwnershipProbe { mutation?: "rotate" | "withdraw"; } -function runOwnershipProbe(probe: OwnershipProbe): Record { +interface CoverageProbeResult { + expected: string; + completeness: string; + warnings: string[]; + openQuestions?: Array<{ question: string }>; +} + +function runPythonProbe(script: string, root: string, probe: unknown): T { const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); - expect(python).not.toBeNull(); if (python === null) { throw new Error("A Python interpreter is required for deep-scan tests."); } const result = Bun.spawnSync( - [ - python, - "-I", - "-B", - "-c", - deepScanOwnershipProbe, - join(PLUGIN_ROOT, "scripts"), - JSON.stringify(probe), - ], + [python, "-I", "-B", "-c", script, root, JSON.stringify(probe)], { stdout: "pipe", stderr: "pipe" }, ); expect(new TextDecoder().decode(result.stderr)).toBe(""); expect(result.exitCode).toBe(0); - return JSON.parse(new TextDecoder().decode(result.stdout)) as Record< - string, - unknown - >; + return JSON.parse(new TextDecoder().decode(result.stdout)) as T; +} + +function runOwnershipProbe(probe: OwnershipProbe): Record { + return runPythonProbe( + deepScanOwnershipProbe, + join(PLUGIN_ROOT, "scripts"), + probe, + ); } test("copies a Deep Scan publication when the filesystem rejects hardlinks", async () => { @@ -284,6 +287,200 @@ test.each([ ); describe("deep scan workbench ownership", () => { + test("restores complete deep coverage only without pending or lost work", () => { + const cases = [ + { expected: "complete" }, + { + expected: "complete", + warnings: ["Recovered finding 1: normalized semantic anchor."], + }, + { + expected: "complete", + warnings: [ + "Recovered finding 2: retained stronger duplicate logical finding.", + ], + }, + { + expected: "complete", + warnings: [ + "Repository HEAD changed while the scan was running; results were saved for the original revision.", + ], + }, + { + expected: "partial", + warnings: ["Recovered finding 1: discarded unverified evidence."], + }, + { + expected: "partial", + warnings: ["Saved checkpoint could not be read."], + }, + { expected: "partial", coverage: { surfaces: [] } }, + { + expected: "partial", + coverage: { + deferred: [{ id: "pending", reason: "Review is incomplete." }], + }, + }, + { + expected: "partial", + coverage: { + surfaces: [ + { + id: "pending", + label: "Pending review", + disposition: "needs_follow_up", + receiptRefs: [], + }, + ], + }, + }, + { + expected: "partial", + coverage: { + surfaces: [ + { + id: "invalid-receipt", + label: "Source review", + disposition: "reported", + receiptRefs: ["artifacts/missing-receipt.json"], + }, + ], + }, + }, + { expected: "partial", coverage: { explicitExclusions: [null] } }, + { expected: "partial", discarded: ["Discarded malformed finding."] }, + { expected: "partial", coverage: { mode: "repository" } }, + { expected: "unknown", coverage: { completeness: "unknown" } }, + ] as const; + const recovered = runPythonProbe( + [ + "import copy, json, pathlib, runpy, sys", + "plugin = pathlib.Path(sys.argv[1])", + "examples = plugin / 'examples' / 'completed-scan'", + "example = json.loads((examples / 'coverage.json').read_text())", + "recover = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))['_recover_unsealed_coverage']", + "results = []", + "for case in json.loads(sys.argv[2]):", + " coverage = copy.deepcopy(example)", + " coverage.update(mode='deep_repository', completeness='partial', openQuestions=[{'question': 'Which deployment controls apply?'}])", + " coverage.update(case.get('coverage', {}))", + " warnings = list(case.get('warnings', []))", + " recover(coverage, plugin / 'schemas', examples, warnings, case.get('discarded', []))", + " results.append({'expected': case['expected'], 'completeness': coverage['completeness'], 'warnings': warnings, 'openQuestions': coverage['openQuestions']})", + "print(json.dumps(results))", + ].join("\n"), + PLUGIN_ROOT, + cases, + ); + for (const result of recovered) { + expect(result.completeness).toBe(result.expected); + expect(result.openQuestions).toEqual([ + { question: "Which deployment controls apply?" }, + ]); + } + }); + + test("preserves worker and parent coverage provenance during recovery", () => { + const cases = [ + ...[null, 0, "false", false].flatMap((parentComplete) => [ + { parentComplete }, + { parent: "complete", parentComplete, retry: true }, + { parentComplete, kind: "dedup" }, + ]), + { parent: "complete", complete: false }, + { complete: false }, + { complete: null }, + { complete: 0 }, + { complete: "false" }, + { parent: "complete", complete: null, retry: true }, + { parent: "complete", complete: 0, retry: true }, + { parent: "complete", complete: "false", retry: true }, + { complete: true, expected: "complete" }, + { parent: "complete", worker: "unknown" }, + { worker: "unknown" }, + { worker: null }, + { worker: "invalid" }, + { worker: [] }, + { worker: "partial", malformed: "deferred" }, + { parent: "complete", malformed: "surfaces" }, + { parent: "complete", worker: "partial" }, + { worker: "partial" }, + { parent: "unknown", worker: "partial" }, + { parent: "unknown", expected: "unknown" }, + { expected: "complete" }, + { missing: true }, + { kind: "dedup", missing: true }, + { parent: "complete", missing: true, retry: true }, + { parent: "complete", kind: "dedup", missing: true, retry: true }, + { status: "failed", missing: true, expected: "complete" }, + { status: "canceled", missing: true, expected: "complete" }, + ] as const; + const recovered = runPythonProbe( + [ + "import copy, json, pathlib, sys", + "plugin = pathlib.Path(sys.argv[1])", + "sys.path.insert(0, str(plugin / 'scripts'))", + "import finalize_scan_contract as finalizer", + "import workbench_saved_results as saved", + "examples = plugin / 'examples' / 'completed-scan'", + "example = json.loads((examples / 'coverage.json').read_text())", + "binding = dict(allowedTargetKinds=['directory_snapshot'],", + " target={'targetId': 'synthetic-target', 'displayName': 'synthetic-repository'},", + " scope={'includePaths': ['.'], 'excludePaths': []},", + " coverageMode='deep_repository', status='completed')", + "saved._children = lambda *_: []", + "saved.write_scan_local_bytes = lambda *_args, **_kwargs: None", + "results = []", + "for case in json.loads(sys.argv[2]):", + " parent_status = case.get('parent', 'partial')", + " worker_complete = case.get('complete', 'omitted')", + " worker_status = case.get('worker', 'complete')", + " malformed = case.get('malformed')", + " expected = case.get('expected', 'partial')", + " workers = []", + " for index in range(2):", + " name = f'synthetic-worker-{index}'", + " workers.append(dict(id=name, kind=case.get('kind', 'discovery'),", + " status=case.get('status', 'succeeded'), artifact_dir=str(examples / name),", + " result_manifest_path=str(examples / name / 'result.json'), completed_at='now', attempt=1))", + " parent_coverage = copy.deepcopy(example)", + " parent_coverage.update(scanId='synthetic-scan', mode='deep_repository', completeness=parent_status)", + " worker_coverage = {'surfaces': [], 'explicitExclusions': [], 'deferred': []}", + " if worker_status is not None: worker_coverage['completeness'] = worker_status", + " if malformed is not None: worker_coverage[malformed] = 'unverified review'", + " parent_scan = dict(id='synthetic-scan', complete=case.get('parentComplete', True), scope=binding['scope'],", + " target={'kind': 'directory_snapshot', **binding['target']})", + " drafts = {'scan-manifest.json': {'scan': parent_scan},", + " 'findings.json': {'findings': []}, 'coverage.json': parent_coverage}", + " worker_draft = {'scanId': 'synthetic-scan', 'findings': [], 'coverage': worker_coverage}", + " if worker_complete != 'omitted': worker_draft['complete'] = worker_complete", + " if not case.get('missing'):", + " for worker in workers: drafts[f\"{worker['id']}/result.json\"] = worker_draft", + " def read_draft(_root, relative, _label):", + " if relative not in drafts: raise FileNotFoundError(relative)", + " return copy.deepcopy(drafts[relative])", + " saved._read_scan_local_json = read_draft", + " warnings = [saved._UNVERIFIED_COVERAGE_WARNING] if case.get('retry') else []", + " coverage = saved.merge_saved_results(examples, 'synthetic-scan', binding, workers, warnings, stopped=False, reason='')[2]", + " finalizer._recover_unsealed_coverage(coverage, plugin / 'schemas', examples, warnings, [])", + " results.append({'expected': expected, 'completeness': coverage['completeness'], 'warnings': warnings})", + "print(json.dumps(results))", + ].join("\n"), + PLUGIN_ROOT, + cases, + ); + for (const result of recovered) { + expect(result.completeness).toBe(result.expected); + expect( + result.warnings.filter( + (warning) => + warning === + "Saved scan source is incomplete or has unverified coverage; coverage remains partial.", + ), + ).toHaveLength(result.expected === "partial" ? 1 : 0); + } + }); + test("starts a Deep Scan with oversized stdin user context", async () => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-context-stdin-")), diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 66fc9bff7..c9fce4f31 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -1952,62 +1952,78 @@ describe("plugin runtime preparation", () => { ]); }); - test("upgrades a cached 0.1.37 plugin with the real bundled Codex executable", async () => { - const root = await temporaryDirectory(); - const previous = await plugin(join(root, "previous"), "0.1.37"); - await writeFile( - join(previous, ".mcp.json"), - JSON.stringify({ mcpServers: { "codex-security": { env_vars: [] } } }), - ); - const home = join(root, "home"); - await mkdir(home, { mode: 0o700 }); - await writeFile( - join(home, "config.toml"), - 'cli_auth_credentials_store = "file"\n\n[features]\nplugins = true\n', - ); - - const command = resolveCodexCommand(); - const environment = { - ...process.env, - CODEX_HOME: home, - OPENAI_API_KEY: undefined, - CODEX_API_KEY: undefined, - }; - const login = spawnSync(command.command, ["login", "--with-api-key"], { - env: environment, - input: "synthetic-key\n", - encoding: "utf8", - windowsHide: true, - }); - expect(login.status).toBe(0); - const credentials = await readFile(join(home, "auth.json"), "utf8"); - - const options = { codexCommand: command, environment }; - const first = await bootstrapPlugin(home, previous, options); - expect(first.version).toBe("0.1.37"); - const upgraded = await bootstrapPlugin(home, PLUGIN_ROOT, options); - const configuration = JSON.parse( - await readFile(join(upgraded.installedRoot, ".mcp.json"), "utf8"), - ) as { - mcpServers: Record; - }; - const server = configuration.mcpServers["codex-security"]; + test.each(["0.1.37", "0.1.59", "0.1.60"])( + "upgrades a cached %s plugin with the real bundled Codex executable", + async (previousVersion) => { + const root = await temporaryDirectory(); + const previous = await plugin(join(root, "previous"), previousVersion); + const validator = "scripts/finalize_scan_contract.py"; + await writeFile( + join(previous, validator), + "# stale synthetic validator\n", + ); + await writeFile( + join(previous, ".mcp.json"), + JSON.stringify({ mcpServers: { "codex-security": { env_vars: [] } } }), + ); + const home = join(root, "home"); + await mkdir(home, { mode: 0o700 }); + await writeFile( + join(home, "config.toml"), + 'cli_auth_credentials_store = "file"\n\n[features]\nplugins = true\n', + ); - expect(upgraded.version).toBe(BUNDLED_PLUGIN_VERSION); - expect(upgraded.version).not.toBe(first.version); - expect(upgraded.installedRoot).not.toBe(first.installedRoot); - expect(server?.command).toBe("./scripts/launch_codex_security_mcp"); - expect(server?.env_vars).toContain("CODEX_MANAGED_PACKAGE_ROOT"); - expect(server?.env_vars).toContain("CODEX_MCP_NODE_PATH"); - expect(await readFile(join(home, "auth.json"), "utf8")).toBe(credentials); - expect( - spawnSync(command.command, ["login", "status"], { + const command = resolveCodexCommand(); + const environment = { + ...process.env, + CODEX_HOME: home, + OPENAI_API_KEY: undefined, + CODEX_API_KEY: undefined, + }; + const login = spawnSync(command.command, ["login", "--with-api-key"], { env: environment, + input: "synthetic-key\n", encoding: "utf8", windowsHide: true, - }).status, - ).toBe(0); - }); + }); + expect(login.status).toBe(0); + const credentials = await readFile(join(home, "auth.json"), "utf8"); + + const options = { codexCommand: command, environment }; + const first = await bootstrapPlugin(home, previous, options); + expect(first.version).toBe(previousVersion); + expect(await readFile(join(first.installedRoot, validator), "utf8")).toBe( + "# stale synthetic validator\n", + ); + const upgraded = await bootstrapPlugin(home, PLUGIN_ROOT, options); + const configuration = JSON.parse( + await readFile(join(upgraded.installedRoot, ".mcp.json"), "utf8"), + ) as { + mcpServers: Record; + }; + const server = configuration.mcpServers["codex-security"]; + + expect(upgraded.version).toBe(BUNDLED_PLUGIN_VERSION); + expect(upgraded.version).not.toBe(first.version); + expect(upgraded.installedRoot).not.toBe(first.installedRoot); + for (const path of [validator, "scripts/workbench_saved_results.py"]) { + expect(await readFile(join(upgraded.installedRoot, path))).toEqual( + await readFile(join(PLUGIN_ROOT, path)), + ); + } + expect(server?.command).toBe("./scripts/launch_codex_security_mcp"); + expect(server?.env_vars).toContain("CODEX_MANAGED_PACKAGE_ROOT"); + expect(server?.env_vars).toContain("CODEX_MCP_NODE_PATH"); + expect(await readFile(join(home, "auth.json"), "utf8")).toBe(credentials); + expect( + spawnSync(command.command, ["login", "status"], { + env: environment, + encoding: "utf8", + windowsHide: true, + }).status, + ).toBe(0); + }, + ); test("resolves the exact npm Codex executable", () => { const command = resolveCodexCommand();