From 6a06171f19e79042f6af81ecaa89c545398e24cc Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 26 Aug 2026 01:47:45 -0400 Subject: [PATCH 1/3] fix(deep-scan): recover complete coverage without masking lost work --- .../scripts/finalize_scan_contract.py | 15 ++ .../scripts/workbench_saved_results.py | 29 ++- .../tests-ts/deep-scan-workbench.test.ts | 167 ++++++++++++++++++ 3 files changed, 202 insertions(+), 9 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index d250e5c91..92bd78b98 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -1023,6 +1023,21 @@ def _recover_unsealed_coverage( partial = True if partial: coverage["completeness"] = "partial" + elif ( + completeness == "partial" + and coverage.get("mode") == "deep_repository" + and coverage["surfaces"] + and not coverage["deferred"] + and not warnings + and not any( + surface["disposition"] == "needs_follow_up" + for surface in coverage["surfaces"] + ) + ): + coverage["completeness"] = "complete" + warnings.append( + "Recovered Deep Scan coverage marked partial without deferred review work." + ) def _recover_unsealed_hardening( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py index 7d4e1c979..b9c618485 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py @@ -577,16 +577,27 @@ def valid_finding(value: Any) -> bool: for saved_path, current, saved_worker in sources ) ) - if ( - (relative != "parent" or not parent_manifest) - and not superseded - and ( + if (relative != "parent" or not parent_manifest) and not superseded: + source_coverage = draft["coverage"] + source_completeness = source_coverage.get("completeness") + if ( draft.get("complete") is False - or draft["coverage"].get("completeness") != "complete" - ) - and coverage.get("completeness") in {"complete", "unknown"} - ): - coverage["completeness"] = "partial" + or source_completeness not in ("complete", "partial") + or any( + not isinstance(source_coverage.get(field), list) + for field in ("surfaces", "explicitExclusions", "deferred") + ) + ): + warning = ( + "Saved scan source is incomplete or has unverified coverage; " + "coverage remains partial." + ) + if warning not in warnings: + warnings.append(warning) + if ( + draft.get("complete") is False or source_completeness != "complete" + ) and coverage.get("completeness") in {"complete", "unknown"}: + coverage["completeness"] = "partial" if superseded and not stopped: continue if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict): diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 02c66b8d6..be8782565 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -284,6 +284,173 @@ test.each([ ); describe("deep scan workbench ownership", () => { + function runCoverageProbe(script: string[], cases: readonly unknown[]) { + const python = Bun.which("python3") ?? Bun.which("python"); + expect(python).not.toBeNull(); + const result = Bun.spawnSync( + [ + python!, + "-I", + "-B", + "-c", + script.join("\n"), + PLUGIN_ROOT, + JSON.stringify(cases), + ], + { stdout: "pipe", stderr: "pipe" }, + ); + expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); + return JSON.parse(new TextDecoder().decode(result.stdout)) as Array<{ + expected: string; + completeness: string; + warnings: string[]; + openQuestions?: Array<{ question: string }>; + }>; + } + + test("restores complete deep coverage only without pending or lost work", () => { + const cases = [ + { expected: "complete" }, + { + expected: "partial", + warnings: ["Saved checkpoint could not be read."], + }, + { expected: "partial", coverage: { surfaces: [] } }, + { + expected: "partial", + coverage: { + deferred: [{ id: "pending", reason: "Review is incomplete." }], + }, + }, + { + expected: "partial", + coverage: { + surfaces: [ + { + id: "pending", + label: "Pending review", + disposition: "needs_follow_up", + receiptRefs: [], + }, + ], + }, + }, + { + expected: "partial", + coverage: { + surfaces: [ + { + id: "invalid-receipt", + label: "Source review", + disposition: "reported", + receiptRefs: ["artifacts/missing-receipt.json"], + }, + ], + }, + }, + { expected: "partial", coverage: { explicitExclusions: [null] } }, + { expected: "partial", discarded: ["Discarded malformed finding."] }, + { expected: "partial", coverage: { mode: "repository" } }, + ] as const; + const recovered = runCoverageProbe( + [ + "import copy, json, pathlib, runpy, sys", + "plugin = pathlib.Path(sys.argv[1])", + "examples = plugin / 'examples' / 'completed-scan'", + "example = json.loads((examples / 'coverage.json').read_text())", + "recover = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))['_recover_unsealed_coverage']", + "results = []", + "for case in json.loads(sys.argv[2]):", + " coverage = copy.deepcopy(example)", + " coverage.update(mode='deep_repository', completeness='partial', openQuestions=[{'question': 'Which deployment controls apply?'}])", + " coverage.update(case.get('coverage', {}))", + " warnings = list(case.get('warnings', []))", + " recover(coverage, plugin / 'schemas', examples, warnings, case.get('discarded', []))", + " results.append({'expected': case['expected'], 'completeness': coverage['completeness'], 'warnings': warnings, 'openQuestions': coverage['openQuestions']})", + "print(json.dumps(results))", + ], + cases, + ); + for (const result of recovered) { + expect(result.completeness).toBe(result.expected); + expect(result.openQuestions).toEqual([ + { question: "Which deployment controls apply?" }, + ]); + } + }); + + test("preserves incomplete worker provenance when recovering deep coverage", () => { + const cases = [ + ["complete", false, "complete", null, "partial"], + ["partial", false, "complete", null, "partial"], + ["complete", null, "unknown", null, "partial"], + ["partial", null, "unknown", null, "partial"], + ["partial", null, null, null, "partial"], + ["partial", null, "invalid", null, "partial"], + ["partial", null, [], null, "partial"], + ["partial", null, "partial", "deferred", "partial"], + ["complete", null, "complete", "surfaces", "partial"], + ["complete", null, "partial", null, "complete"], + ["partial", null, "partial", null, "complete"], + ["partial", null, "complete", null, "complete"], + ] as const; + const recovered = runCoverageProbe( + [ + "import copy, json, pathlib, sys", + "plugin = pathlib.Path(sys.argv[1])", + "sys.path.insert(0, str(plugin / 'scripts'))", + "import finalize_scan_contract as finalizer", + "import workbench_saved_results as saved", + "examples = plugin / 'examples' / 'completed-scan'", + "example = json.loads((examples / 'coverage.json').read_text())", + "binding = dict(allowedTargetKinds=['directory_snapshot'],", + " target={'targetId': 'synthetic-target', 'displayName': 'synthetic-repository'},", + " scope={'includePaths': ['.'], 'excludePaths': []},", + " coverageMode='deep_repository', status='completed')", + "workers = []", + "for index in range(2):", + " name = f'synthetic-worker-{index}'", + " workers.append(dict(id=name, kind='discovery', status='succeeded',", + " artifact_dir=str(examples / name),", + " result_manifest_path=str(examples / name / 'result.json'),", + " completed_at='now', attempt=1))", + "saved._children = lambda *_: []", + "saved.write_scan_local_bytes = lambda *_args, **_kwargs: None", + "results = []", + "for parent_status, worker_complete, worker_status, malformed, expected in json.loads(sys.argv[2]):", + " parent_coverage = copy.deepcopy(example)", + " parent_coverage.update(scanId='synthetic-scan', mode='deep_repository', completeness=parent_status)", + " worker_coverage = {'surfaces': [], 'explicitExclusions': [], 'deferred': []}", + " if worker_status is not None: worker_coverage['completeness'] = worker_status", + " if malformed is not None: worker_coverage[malformed] = 'unverified review'", + " parent_scan = dict(id='synthetic-scan', complete=True, scope=binding['scope'],", + " target={'kind': 'directory_snapshot', **binding['target']})", + " drafts = {'scan-manifest.json': {'scan': parent_scan},", + " 'findings.json': {'findings': []}, 'coverage.json': parent_coverage}", + " worker_draft = {'scanId': 'synthetic-scan', 'findings': [], 'coverage': worker_coverage}", + " if worker_complete is not None: worker_draft['complete'] = worker_complete", + " for worker in workers: drafts[f\"{worker['id']}/result.json\"] = worker_draft", + " saved._read_scan_local_json = lambda _root, relative, _label: copy.deepcopy(drafts[relative])", + " warnings = []", + " coverage = saved.merge_saved_results(examples, 'synthetic-scan', binding, workers, warnings, stopped=False, reason='')[2]", + " finalizer._recover_unsealed_coverage(coverage, plugin / 'schemas', examples, warnings, [])", + " results.append({'expected': expected, 'completeness': coverage['completeness'], 'warnings': warnings})", + "print(json.dumps(results))", + ], + cases, + ); + for (const result of recovered) { + expect(result.completeness).toBe(result.expected); + expect( + result.warnings.filter( + (warning) => + warning === + "Saved scan source is incomplete or has unverified coverage; coverage remains partial.", + ), + ).toHaveLength(result.expected === "partial" ? 1 : 0); + } + }); + test("starts a Deep Scan with oversized stdin user context", async () => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-context-stdin-")), From c1d997a12f786cf74043efb62b5566e2332986e5 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 26 Aug 2026 02:09:22 -0400 Subject: [PATCH 2/3] fix(deep-scan): preserve worker coverage provenance on retries --- .../scripts/finalize_scan_contract.py | 22 ++- .../scripts/workbench_saved_results.py | 31 +++- .../tests-ts/deep-scan-workbench.test.ts | 136 +++++++++++++++--- 3 files changed, 158 insertions(+), 31 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index 92bd78b98..d24c2d0ff 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -44,6 +44,18 @@ "directory_snapshot": {"snapshotDigest"}, } DISPOSITIONS = {"reported", "no_issue_found", "rejected", "not_applicable", "needs_follow_up"} +NON_COVERAGE_TARGET_WARNINGS = { + "Directory contents changed while the scan was running; " + "results were saved for the original snapshot.", + "The scanned Git repository became unavailable while the scan was running; " + "results were saved for the original revision.", + "Repository HEAD changed while the scan was running; " + "results were saved for the original revision.", + "Working-tree contents changed while the scan was running; " + "results were saved for the original snapshot.", + "The scan target became unavailable while the scan was running; " + "results were saved for the original revision or snapshot.", +} SARIF_LEVELS = { "critical": "error", "high": "error", @@ -1028,7 +1040,15 @@ def _recover_unsealed_coverage( and coverage.get("mode") == "deep_repository" and coverage["surfaces"] and not coverage["deferred"] - and not warnings + and all( + warning in NON_COVERAGE_TARGET_WARNINGS + or re.fullmatch( + r"Recovered finding [0-9]+: " + r"(?:normalized [a-z, ]+|retained stronger duplicate logical finding)\.", + warning, + ) + for warning in warnings + ) and not any( surface["disposition"] == "needs_follow_up" for surface in coverage["surfaces"] diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py index b9c618485..c775b481d 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py @@ -47,6 +47,9 @@ _PUBLICATION_FOLLOW_UP_WARNING = ( "Saved scan evidence remains on disk; result publication needs follow-up:" ) +_UNVERIFIED_COVERAGE_WARNING = ( + "Saved scan source is incomplete or has unverified coverage; coverage remains partial." +) @dataclass(frozen=True) @@ -277,6 +280,7 @@ def merge_saved_results( source_digests.update(parent_preserved_sources) paths: dict[str, str | None] = {} current_results: set[str] = set() + required_results: set[str] = set() reducer_outputs: list[tuple[Any, str, list[str], int]] = [] accepted_reducers = [ worker @@ -293,6 +297,7 @@ def merge_saved_results( try: latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[latest_reducer] = None + required_results.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") @@ -351,6 +356,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[current_path] = worker["id"] current_results.add(current_path) + if worker["status"] == "succeeded": + required_results.add(current_path) except ValueError: warnings.append("Skipped a worker result outside the scan directory.") @@ -381,6 +388,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}") + elif relative in required_results and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if frozen_source_digests is not None: if frozen_source_digests.keys() - source_digests.keys(): raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") @@ -582,18 +591,22 @@ def valid_finding(value: Any) -> bool: source_completeness = source_coverage.get("completeness") if ( draft.get("complete") is False + or ("complete" in draft and not isinstance(draft["complete"], bool)) or source_completeness not in ("complete", "partial") + or ( + source_completeness != "complete" + and ( + worker_id is not None + or coverage.get("completeness") == "unknown" + ) + ) or any( not isinstance(source_coverage.get(field), list) for field in ("surfaces", "explicitExclusions", "deferred") ) ): - warning = ( - "Saved scan source is incomplete or has unverified coverage; " - "coverage remains partial." - ) - if warning not in warnings: - warnings.append(warning) + if _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if ( draft.get("complete") is False or source_completeness != "complete" ) and coverage.get("completeness") in {"complete", "unknown"}: @@ -824,7 +837,11 @@ def valid_finding(value: Any) -> bool: used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) - if stopped or any(warning not in initial_warnings for warning in warnings): + if ( + stopped + or _UNVERIFIED_COVERAGE_WARNING in warnings + or any(warning not in initial_warnings for warning in warnings) + ): coverage["completeness"] = "partial" if stopped: if not isinstance(coverage.get("deferred"), list): diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index be8782565..67e461141 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -311,6 +311,26 @@ describe("deep scan workbench ownership", () => { test("restores complete deep coverage only without pending or lost work", () => { const cases = [ { expected: "complete" }, + { + expected: "complete", + warnings: ["Recovered finding 1: normalized semantic anchor."], + }, + { + expected: "complete", + warnings: [ + "Recovered finding 2: retained stronger duplicate logical finding.", + ], + }, + { + expected: "complete", + warnings: [ + "Repository HEAD changed while the scan was running; results were saved for the original revision.", + ], + }, + { + expected: "partial", + warnings: ["Recovered finding 1: discarded unverified evidence."], + }, { expected: "partial", warnings: ["Saved checkpoint could not be read."], @@ -351,6 +371,7 @@ describe("deep scan workbench ownership", () => { { expected: "partial", coverage: { explicitExclusions: [null] } }, { expected: "partial", discarded: ["Discarded malformed finding."] }, { expected: "partial", coverage: { mode: "repository" } }, + { expected: "unknown", coverage: { completeness: "unknown" } }, ] as const; const recovered = runCoverageProbe( [ @@ -379,20 +400,84 @@ describe("deep scan workbench ownership", () => { } }); - test("preserves incomplete worker provenance when recovering deep coverage", () => { + test("preserves worker and parent coverage provenance during recovery", () => { const cases = [ ["complete", false, "complete", null, "partial"], ["partial", false, "complete", null, "partial"], - ["complete", null, "unknown", null, "partial"], - ["partial", null, "unknown", null, "partial"], - ["partial", null, null, null, "partial"], - ["partial", null, "invalid", null, "partial"], - ["partial", null, [], null, "partial"], - ["partial", null, "partial", "deferred", "partial"], - ["complete", null, "complete", "surfaces", "partial"], - ["complete", null, "partial", null, "complete"], - ["partial", null, "partial", null, "complete"], - ["partial", null, "complete", null, "complete"], + ["partial", null, "complete", null, "partial"], + ["partial", 0, "complete", null, "partial"], + ["partial", "false", "complete", null, "partial"], + ["complete", null, "complete", null, "partial", { priorWarning: true }], + ["complete", 0, "complete", null, "partial", { priorWarning: true }], + [ + "complete", + "false", + "complete", + null, + "partial", + { priorWarning: true }, + ], + ["partial", true, "complete", null, "complete"], + ["complete", "omitted", "unknown", null, "partial"], + ["partial", "omitted", "unknown", null, "partial"], + ["partial", "omitted", null, null, "partial"], + ["partial", "omitted", "invalid", null, "partial"], + ["partial", "omitted", [], null, "partial"], + ["partial", "omitted", "partial", "deferred", "partial"], + ["complete", "omitted", "complete", "surfaces", "partial"], + ["complete", "omitted", "partial", null, "partial"], + ["partial", "omitted", "partial", null, "partial"], + ["unknown", "omitted", "partial", null, "partial"], + ["unknown", "omitted", "complete", null, "unknown"], + ["partial", "omitted", "complete", null, "complete"], + [ + "partial", + "omitted", + "complete", + null, + "partial", + { kind: "discovery", missing: true }, + ], + [ + "partial", + "omitted", + "complete", + null, + "partial", + { kind: "dedup", missing: true }, + ], + [ + "complete", + "omitted", + "complete", + null, + "partial", + { kind: "discovery", missing: true, priorWarning: true }, + ], + [ + "complete", + "omitted", + "complete", + null, + "partial", + { kind: "dedup", missing: true, priorWarning: true }, + ], + [ + "partial", + "omitted", + "complete", + null, + "complete", + { kind: "discovery", status: "failed", missing: true }, + ], + [ + "partial", + "omitted", + "complete", + null, + "complete", + { kind: "discovery", status: "canceled", missing: true }, + ], ] as const; const recovered = runCoverageProbe( [ @@ -407,17 +492,18 @@ describe("deep scan workbench ownership", () => { " target={'targetId': 'synthetic-target', 'displayName': 'synthetic-repository'},", " scope={'includePaths': ['.'], 'excludePaths': []},", " coverageMode='deep_repository', status='completed')", - "workers = []", - "for index in range(2):", - " name = f'synthetic-worker-{index}'", - " workers.append(dict(id=name, kind='discovery', status='succeeded',", - " artifact_dir=str(examples / name),", - " result_manifest_path=str(examples / name / 'result.json'),", - " completed_at='now', attempt=1))", "saved._children = lambda *_: []", "saved.write_scan_local_bytes = lambda *_args, **_kwargs: None", "results = []", - "for parent_status, worker_complete, worker_status, malformed, expected in json.loads(sys.argv[2]):", + "for case in json.loads(sys.argv[2]):", + " parent_status, worker_complete, worker_status, malformed, expected = case[:5]", + " options = case[5] if len(case) > 5 else {}", + " workers = []", + " for index in range(2):", + " name = f'synthetic-worker-{index}'", + " workers.append(dict(id=name, kind=options.get('kind', 'discovery'),", + " status=options.get('status', 'succeeded'), artifact_dir=str(examples / name),", + " result_manifest_path=str(examples / name / 'result.json'), completed_at='now', attempt=1))", " parent_coverage = copy.deepcopy(example)", " parent_coverage.update(scanId='synthetic-scan', mode='deep_repository', completeness=parent_status)", " worker_coverage = {'surfaces': [], 'explicitExclusions': [], 'deferred': []}", @@ -428,10 +514,14 @@ describe("deep scan workbench ownership", () => { " drafts = {'scan-manifest.json': {'scan': parent_scan},", " 'findings.json': {'findings': []}, 'coverage.json': parent_coverage}", " worker_draft = {'scanId': 'synthetic-scan', 'findings': [], 'coverage': worker_coverage}", - " if worker_complete is not None: worker_draft['complete'] = worker_complete", - " for worker in workers: drafts[f\"{worker['id']}/result.json\"] = worker_draft", - " saved._read_scan_local_json = lambda _root, relative, _label: copy.deepcopy(drafts[relative])", - " warnings = []", + " if worker_complete != 'omitted': worker_draft['complete'] = worker_complete", + " if not options.get('missing'):", + " for worker in workers: drafts[f\"{worker['id']}/result.json\"] = worker_draft", + " def read_draft(_root, relative, _label):", + " if relative not in drafts: raise FileNotFoundError(relative)", + " return copy.deepcopy(drafts[relative])", + " saved._read_scan_local_json = read_draft", + " warnings = [saved._UNVERIFIED_COVERAGE_WARNING] if options.get('priorWarning') else []", " coverage = saved.merge_saved_results(examples, 'synthetic-scan', binding, workers, warnings, stopped=False, reason='')[2]", " finalizer._recover_unsealed_coverage(coverage, plugin / 'schemas', examples, warnings, [])", " results.append({'expected': expected, 'completeness': coverage['completeness'], 'warnings': warnings})", From 2db25481cdd2835e187e5939a6c850aa7364bdf7 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 26 Aug 2026 02:24:04 -0400 Subject: [PATCH 3/3] refactor(deep-scan): simplify coverage recovery and regression probes --- .../scripts/finalize_scan_contract.py | 4 - .../scripts/workbench_saved_results.py | 13 +- .../tests-ts/deep-scan-workbench.test.ts | 184 ++++++------------ 3 files changed, 66 insertions(+), 135 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index d24c2d0ff..0e8a1bd80 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -1049,10 +1049,6 @@ def _recover_unsealed_coverage( ) for warning in warnings ) - and not any( - surface["disposition"] == "needs_follow_up" - for surface in coverage["surfaces"] - ) ): coverage["completeness"] = "complete" warnings.append( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py index c775b481d..fd4f4848f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_saved_results.py @@ -589,14 +589,14 @@ def valid_finding(value: Any) -> bool: if (relative != "parent" or not parent_manifest) and not superseded: source_coverage = draft["coverage"] source_completeness = source_coverage.get("completeness") + source_complete = draft.get("complete", True) is True if ( - draft.get("complete") is False - or ("complete" in draft and not isinstance(draft["complete"], bool)) - or source_completeness not in ("complete", "partial") + not source_complete or ( source_completeness != "complete" and ( worker_id is not None + or source_completeness != "partial" or coverage.get("completeness") == "unknown" ) ) @@ -604,11 +604,10 @@ def valid_finding(value: Any) -> bool: not isinstance(source_coverage.get(field), list) for field in ("surfaces", "explicitExclusions", "deferred") ) - ): - if _UNVERIFIED_COVERAGE_WARNING not in warnings: - warnings.append(_UNVERIFIED_COVERAGE_WARNING) + ) and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if ( - draft.get("complete") is False or source_completeness != "complete" + not source_complete or source_completeness != "complete" ) and coverage.get("completeness") in {"complete", "unknown"}: coverage["completeness"] = "partial" if superseded and not stopped: diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 67e461141..f74c73d7b 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -66,7 +66,14 @@ interface OwnershipProbe { mutation?: "rotate" | "withdraw"; } -function runOwnershipProbe(probe: OwnershipProbe): Record { +interface CoverageProbeResult { + expected: string; + completeness: string; + warnings: string[]; + openQuestions?: Array<{ question: string }>; +} + +function runPythonProbe(script: string, root: string, probe: unknown): T { const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); expect(python).not.toBeNull(); if (python === null) { @@ -74,23 +81,20 @@ function runOwnershipProbe(probe: OwnershipProbe): Record { } const result = Bun.spawnSync( - [ - python, - "-I", - "-B", - "-c", - deepScanOwnershipProbe, - join(PLUGIN_ROOT, "scripts"), - JSON.stringify(probe), - ], + [python, "-I", "-B", "-c", script, root, JSON.stringify(probe)], { stdout: "pipe", stderr: "pipe" }, ); expect(new TextDecoder().decode(result.stderr)).toBe(""); expect(result.exitCode).toBe(0); - return JSON.parse(new TextDecoder().decode(result.stdout)) as Record< - string, - unknown - >; + return JSON.parse(new TextDecoder().decode(result.stdout)) as T; +} + +function runOwnershipProbe(probe: OwnershipProbe): Record { + return runPythonProbe( + deepScanOwnershipProbe, + join(PLUGIN_ROOT, "scripts"), + probe, + ); } test("copies a Deep Scan publication when the filesystem rejects hardlinks", async () => { @@ -284,30 +288,6 @@ test.each([ ); describe("deep scan workbench ownership", () => { - function runCoverageProbe(script: string[], cases: readonly unknown[]) { - const python = Bun.which("python3") ?? Bun.which("python"); - expect(python).not.toBeNull(); - const result = Bun.spawnSync( - [ - python!, - "-I", - "-B", - "-c", - script.join("\n"), - PLUGIN_ROOT, - JSON.stringify(cases), - ], - { stdout: "pipe", stderr: "pipe" }, - ); - expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); - return JSON.parse(new TextDecoder().decode(result.stdout)) as Array<{ - expected: string; - completeness: string; - warnings: string[]; - openQuestions?: Array<{ question: string }>; - }>; - } - test("restores complete deep coverage only without pending or lost work", () => { const cases = [ { expected: "complete" }, @@ -373,7 +353,7 @@ describe("deep scan workbench ownership", () => { { expected: "partial", coverage: { mode: "repository" } }, { expected: "unknown", coverage: { completeness: "unknown" } }, ] as const; - const recovered = runCoverageProbe( + const recovered = runPythonProbe( [ "import copy, json, pathlib, runpy, sys", "plugin = pathlib.Path(sys.argv[1])", @@ -389,7 +369,8 @@ describe("deep scan workbench ownership", () => { " recover(coverage, plugin / 'schemas', examples, warnings, case.get('discarded', []))", " results.append({'expected': case['expected'], 'completeness': coverage['completeness'], 'warnings': warnings, 'openQuestions': coverage['openQuestions']})", "print(json.dumps(results))", - ], + ].join("\n"), + PLUGIN_ROOT, cases, ); for (const result of recovered) { @@ -402,84 +383,35 @@ describe("deep scan workbench ownership", () => { test("preserves worker and parent coverage provenance during recovery", () => { const cases = [ - ["complete", false, "complete", null, "partial"], - ["partial", false, "complete", null, "partial"], - ["partial", null, "complete", null, "partial"], - ["partial", 0, "complete", null, "partial"], - ["partial", "false", "complete", null, "partial"], - ["complete", null, "complete", null, "partial", { priorWarning: true }], - ["complete", 0, "complete", null, "partial", { priorWarning: true }], - [ - "complete", - "false", - "complete", - null, - "partial", - { priorWarning: true }, - ], - ["partial", true, "complete", null, "complete"], - ["complete", "omitted", "unknown", null, "partial"], - ["partial", "omitted", "unknown", null, "partial"], - ["partial", "omitted", null, null, "partial"], - ["partial", "omitted", "invalid", null, "partial"], - ["partial", "omitted", [], null, "partial"], - ["partial", "omitted", "partial", "deferred", "partial"], - ["complete", "omitted", "complete", "surfaces", "partial"], - ["complete", "omitted", "partial", null, "partial"], - ["partial", "omitted", "partial", null, "partial"], - ["unknown", "omitted", "partial", null, "partial"], - ["unknown", "omitted", "complete", null, "unknown"], - ["partial", "omitted", "complete", null, "complete"], - [ - "partial", - "omitted", - "complete", - null, - "partial", - { kind: "discovery", missing: true }, - ], - [ - "partial", - "omitted", - "complete", - null, - "partial", - { kind: "dedup", missing: true }, - ], - [ - "complete", - "omitted", - "complete", - null, - "partial", - { kind: "discovery", missing: true, priorWarning: true }, - ], - [ - "complete", - "omitted", - "complete", - null, - "partial", - { kind: "dedup", missing: true, priorWarning: true }, - ], - [ - "partial", - "omitted", - "complete", - null, - "complete", - { kind: "discovery", status: "failed", missing: true }, - ], - [ - "partial", - "omitted", - "complete", - null, - "complete", - { kind: "discovery", status: "canceled", missing: true }, - ], + { parent: "complete", complete: false }, + { complete: false }, + { complete: null }, + { complete: 0 }, + { complete: "false" }, + { parent: "complete", complete: null, retry: true }, + { parent: "complete", complete: 0, retry: true }, + { parent: "complete", complete: "false", retry: true }, + { complete: true, expected: "complete" }, + { parent: "complete", worker: "unknown" }, + { worker: "unknown" }, + { worker: null }, + { worker: "invalid" }, + { worker: [] }, + { worker: "partial", malformed: "deferred" }, + { parent: "complete", malformed: "surfaces" }, + { parent: "complete", worker: "partial" }, + { worker: "partial" }, + { parent: "unknown", worker: "partial" }, + { parent: "unknown", expected: "unknown" }, + { expected: "complete" }, + { missing: true }, + { kind: "dedup", missing: true }, + { parent: "complete", missing: true, retry: true }, + { parent: "complete", kind: "dedup", missing: true, retry: true }, + { status: "failed", missing: true, expected: "complete" }, + { status: "canceled", missing: true, expected: "complete" }, ] as const; - const recovered = runCoverageProbe( + const recovered = runPythonProbe( [ "import copy, json, pathlib, sys", "plugin = pathlib.Path(sys.argv[1])", @@ -496,13 +428,16 @@ describe("deep scan workbench ownership", () => { "saved.write_scan_local_bytes = lambda *_args, **_kwargs: None", "results = []", "for case in json.loads(sys.argv[2]):", - " parent_status, worker_complete, worker_status, malformed, expected = case[:5]", - " options = case[5] if len(case) > 5 else {}", + " parent_status = case.get('parent', 'partial')", + " worker_complete = case.get('complete', 'omitted')", + " worker_status = case.get('worker', 'complete')", + " malformed = case.get('malformed')", + " expected = case.get('expected', 'partial')", " workers = []", " for index in range(2):", " name = f'synthetic-worker-{index}'", - " workers.append(dict(id=name, kind=options.get('kind', 'discovery'),", - " status=options.get('status', 'succeeded'), artifact_dir=str(examples / name),", + " workers.append(dict(id=name, kind=case.get('kind', 'discovery'),", + " status=case.get('status', 'succeeded'), artifact_dir=str(examples / name),", " result_manifest_path=str(examples / name / 'result.json'), completed_at='now', attempt=1))", " parent_coverage = copy.deepcopy(example)", " parent_coverage.update(scanId='synthetic-scan', mode='deep_repository', completeness=parent_status)", @@ -515,18 +450,19 @@ describe("deep scan workbench ownership", () => { " 'findings.json': {'findings': []}, 'coverage.json': parent_coverage}", " worker_draft = {'scanId': 'synthetic-scan', 'findings': [], 'coverage': worker_coverage}", " if worker_complete != 'omitted': worker_draft['complete'] = worker_complete", - " if not options.get('missing'):", + " if not case.get('missing'):", " for worker in workers: drafts[f\"{worker['id']}/result.json\"] = worker_draft", " def read_draft(_root, relative, _label):", " if relative not in drafts: raise FileNotFoundError(relative)", " return copy.deepcopy(drafts[relative])", " saved._read_scan_local_json = read_draft", - " warnings = [saved._UNVERIFIED_COVERAGE_WARNING] if options.get('priorWarning') else []", + " warnings = [saved._UNVERIFIED_COVERAGE_WARNING] if case.get('retry') else []", " coverage = saved.merge_saved_results(examples, 'synthetic-scan', binding, workers, warnings, stopped=False, reason='')[2]", " finalizer._recover_unsealed_coverage(coverage, plugin / 'schemas', examples, warnings, [])", " results.append({'expected': expected, 'completeness': coverage['completeness'], 'warnings': warnings})", "print(json.dumps(results))", - ], + ].join("\n"), + PLUGIN_ROOT, cases, ); for (const result of recovered) {