From c5548a4950e1440b1dbbcd35a5dda10840879997 Mon Sep 17 00:00:00 2001 From: Deploy Date: Mon, 31 Aug 2026 01:26:16 +0000 Subject: [PATCH 1/3] feat: prototype affected verification planning --- .github/workflows/ci.yml | 31 ++++ .gitignore | 4 + ARCHITECTURE.md | 66 +++++++++ BENCHMARK.md | 65 +++++++++ CONTRIBUTING.md | 4 + LIMITATIONS.md | 15 ++ PRIOR_ART.md | 28 ++++ README.md | 39 ++++- SECURITY.md | 6 + SPEC.md | 97 +++++++++++++ bin/affected-verification.js | 72 ++++++++++ fixtures/negative-cases.js | 32 +++++ fixtures/scenarios.js | 164 +++++++++++++++++++++ package.json | 22 +++ schemas/plan-v1.schema.json | 91 ++++++++++++ src/canonical.js | 35 +++++ src/index.js | 12 ++ src/planner.js | 264 ++++++++++++++++++++++++++++++++++ src/shadow.js | 72 ++++++++++ src/validate.js | 267 +++++++++++++++++++++++++++++++++++ src/value-receipt.js | 96 +++++++++++++ tests/cli.test.js | 47 ++++++ tests/planner.test.js | 204 ++++++++++++++++++++++++++ tests/value-shadow.test.js | 120 ++++++++++++++++ tools/conformance.js | 29 ++++ 25 files changed, 1880 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 ARCHITECTURE.md create mode 100644 BENCHMARK.md create mode 100644 CONTRIBUTING.md create mode 100644 LIMITATIONS.md create mode 100644 PRIOR_ART.md create mode 100644 SECURITY.md create mode 100644 SPEC.md create mode 100755 bin/affected-verification.js create mode 100644 fixtures/negative-cases.js create mode 100644 fixtures/scenarios.js create mode 100644 package.json create mode 100644 schemas/plan-v1.schema.json create mode 100644 src/canonical.js create mode 100644 src/index.js create mode 100644 src/planner.js create mode 100644 src/shadow.js create mode 100644 src/validate.js create mode 100644 src/value-receipt.js create mode 100644 tests/cli.test.js create mode 100644 tests/planner.test.js create mode 100644 tests/value-shadow.test.js create mode 100755 tools/conformance.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..78574c2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm run verify + + secret-scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: gitleaks/gitleaks-action@v2 + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..35e3dfa --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +coverage/ +*.tmp + diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..a2a1282 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,66 @@ +# Architecture and boundaries + +```text +change + normalized evidence + catalog + policy + | + v + deterministic verification planner + | + v + plan: selected + skipped + argument + state + | + execution belongs to an external system +``` + +## Core and adapters + +The core owns validation, reverse-impact closure, deterministic policy matching, fail-closed escalation, selection/skip arguments, canonical hashing, Visible Value planning counts, and shadow-result classification. + +Adapters may translate Git diffs, package/project graphs, imports, coverage, test selectors, CODEOWNERS, schemas, or declared critical boundaries into normalized input. V1 implements no production adapter. This prevents the core from embedding Nx, Turbo, Jest, Vitest, testmon, or any one repository layout. + +The verification catalog is the planner's universe. It is not a scheduler. Commands are opaque identities; the planner never shells out to them. + +## Independence from Gearbox + +Affected Verification owns the question: “What is the minimum defensible verification workload for this change under current evidence and policy?” Gearbox may ask that question and choose where or how to execute the returned work. Gearbox does not own the theory, catalog, or planner. + +The prototype has no Gearbox dependency and is independently usable by people, coding agents, CI, PR bots, Opsle Tasks, or other tooling. + +## Context Firewall boundary + +Affected Verification decides **what should execute**. After execution, Context Firewall decides **what result should enter model context**. For example, the planner may require 17 of 1,000 tests; after those run, Context Firewall may retain only aggregate success plus one failure. Neither mechanism substitutes for the other. + +## Decision Evidence and Trajectory boundaries + +Decision Evidence may later validate change identity, evidence provenance, plan identity, selected/skip arguments, and execution-result bindings. Agent Trajectory Profiler may measure planned selections, actual executions, shadow comparisons, and observed latency/cost when genuinely recorded. V1 records compatible identities but creates no cross-repository package coupling. + +## Shadow observations + +`classifyShadow` first recomputes and verifies the plan identity, then compares a predicted plan with an externally supplied full-run record. Its durable observation binds: + +- plan, change, policy, and planner schema identities; +- predicted selected and skipped check IDs; +- full-run executed IDs and failures; +- whether full execution was complete; +- relevant failures in skipped checks and their exact supplied reason; +- `SELECTION_MISS`, `NO_SELECTION_MISS`, or `INDETERMINATE_FULL_RUN_INCOMPLETE`. + +A selection miss occurs only when a check omitted by the targeted plan fails in full execution and that failure is classified as relevant to the change. Relevance is an external oracle claim in v1; the classifier records it and does not infer causality. + +No telemetry service exists. The fixture-level classifier proves the observation contract only. + +## Trust ramp + +`OBSERVE` +: Generate plans for inspection; existing verification remains authoritative. + +`SHADOW` +: Generate targeted plans while full verification runs; retain complete, identity-bound miss observations. + +`TRUSTED_BOUNDED` +: Permit targeted plans only for repository-specific low-risk classes whose impact/catalog completeness, policy behavior, miss history, oracle quality, and rollback path are defended. + +`TRUSTED_POLICY` +: Extend authority to additional explicitly bounded classes only after their evidence is comparable and their promotion criteria are met. + +Promotion depends on evidence quality and coverage, not an arbitrary run count. Critical/global classes may permanently require full verification. A miss, evidence drift, adapter change, catalog drift, policy change, or loss of shadow completeness can demote trust. diff --git a/BENCHMARK.md b/BENCHMARK.md new file mode 100644 index 0000000..e1d33bf --- /dev/null +++ b/BENCHMARK.md @@ -0,0 +1,65 @@ +# Benchmark and research plan + +No benchmark has run and no result is claimed. + +## Central question + +Can Affected Verification substantially reduce verification workload without increasing undetected regression risk relative to existing full verification? + +Correctness and selection-miss evidence are evaluated before computation reduction. + +## Arms + +For each frozen repository/change fixture where applicable: + +1. full verification; +2. native related/affected tooling; +3. Affected Verification using only normalized repository evidence; +4. Affected Verification consuming native tooling as evidence. + +Native arms should include the repository's actual Jest, Vitest, Nx, Turbo, testmon, Bazel, Pants, or other selector rather than a synthetic strawman. + +## Required freeze before a run + +- exact repository and base/target revisions; +- immutable change set and changed-region derivation; +- complete verification catalog and units; +- evidence-provider versions and outputs; +- policy identity and trust stage; +- commands, environment, and deterministic plan outputs; +- full-verification correctness oracle; +- relevance adjudication protocol for full-run failures; +- failure classes, stopping rules, and excluded changes. + +## Primary safety metrics + +- relevant selection misses found by full/shadow execution; +- incomplete or indeterminate full runs; +- targeted plans that escalated because sufficiency was indefensible; +- false targeted-sufficiency claims discovered by audit; +- correctness-oracle disagreements. + +Any relevant selection miss is reported before reduction. “No miss observed” is bounded to the executed corpus and is not proof of safety. + +## Workload metrics + +- checks available, selected, skipped, and actually executed, partitioned by check type; +- test executions available, selected, skipped, and actually executed; +- computation, latency, and cost only when directly observed in comparable arms; +- uncertainty/escalation frequency by change class; +- marginal reduction from each evidence source by controlled ablation. + +Unlike verification types are not added into one deceptive percentage. Counterfactual avoided computation requires controlled comparability; plan counts alone are `EXACT`, not causal savings. + +## Secondary questions + +- Which evidence sources materially reduce workload? +- How often does uncertainty force escalation? +- Which change classes remain defensibly targetable? +- How frequently do native and composed selectors miss relevant full-run failures? +- Does adding policy and non-test checks produce value above native selectors? + +## Initial experiment sequence + +First, freeze a real public repository with a trustworthy full-verification baseline and run in `OBSERVE`/`SHADOW`. Compare plan identities and full outcomes without replacing CI. Only after the oracle, catalog completeness, adapter fidelity, and miss classifications are independently reviewable should a bounded trust decision be considered. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f79956b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,4 @@ +# Contributing + +Keep the core deterministic, dependency-light, and separate from execution. Changes to selection semantics require exact tests, updated contract text, and new or revised conformance fixtures. Prior-art claims require primary sources. Do not add benchmark results without immutable inputs, a full-verification oracle, and explicit failure reporting. + diff --git a/LIMITATIONS.md b/LIMITATIONS.md new file mode 100644 index 0000000..c4cf3c5 --- /dev/null +++ b/LIMITATIONS.md @@ -0,0 +1,15 @@ +# Limitations and non-goals + +- The prototype consumes synthetic normalized evidence; it has no Git, graph, coverage, ownership, schema, Nx, Turbo, Jest, Vitest, or testmon adapter. +- The component graph and catalog completeness flags are caller claims, not independently attested. +- The selection algorithm uses declared component scope and policy tags. It has no symbol/data-flow analysis, runtime coverage collection, weighted set cover, probabilistic model, or learned judgment. +- Commands are opaque identities. The project does not execute, schedule, cache, distribute, retry, or report CI work. +- `SUFFICIENT_*` means sufficient under the supplied model and policy, not globally safe, formally sound, or mathematically minimal. +- Full catalog selection cannot compensate for an incomplete catalog; that state remains `INSUFFICIENT_EVIDENCE`. +- Shadow relevance is caller supplied. A robust real benchmark needs an independent, reproducible relevance oracle. +- Test execution counts are declared catalog metadata. They are exact relative to input, not observed executions. +- No real-project benchmark, comparative baseline, selection-miss corpus, computation measurement, or independent reproduction exists. +- No production trust stage is justified. This repository is at most `PROTOTYPED`. +- No time, token, monetary, correctness, failure-prevention, or causal savings claim is supported. +- Context Firewall, Decision Evidence, Agent Trajectory Profiler, Gearbox, and Opsle Tasks are external consumers or validators, not dependencies. + diff --git a/PRIOR_ART.md b/PRIOR_ART.md new file mode 100644 index 0000000..6410697 --- /dev/null +++ b/PRIOR_ART.md @@ -0,0 +1,28 @@ +# Prior-art reconciliation + +This audit uses project-owned documentation or source and established research literature. It is a boundary analysis, not a novelty claim. Several existing systems already provide sophisticated affected selection, conservative fallbacks, explanations, and shadow prediction. Affected Verification should reuse them as evidence providers where they are authoritative. + +## Comparison + +| System | Input evidence | Selection unit | Static/runtime | Uncertainty behavior | Non-test verification | Skip explanation | Shadow validation | Sufficiency claim | Intended reuse | +|---|---|---|---|---|---|---|---|---|---| +| [Nx affected](https://nx.dev/docs/features/ci-features/affected) | Git base/head or supplied files, project graph, source/config analysis, lockfile analysis | Projects, then requested targets/tasks | Primarily static workspace and project/task graph | Conservative lockfile default marks all projects; graph/plugin behavior can broaden | Yes: any Nx target such as lint, test, build, or custom task | Affected graph and task graph explain inclusion; no first-class reason for every omitted task | No general OSS CLI shadow/full miss contract found | Claims a minimum affected project set, not acceptance sufficiency across verification classes | Consume affected projects/task graph and its failsafe signals | +| [Turborepo `--affected`](https://github.com/vercel/turborepo/blob/main/apps/docs/content/docs/reference/configuration.mdx) | Git range, package graph, global dependencies, optionally task input globs | Packages by default; tasks with `affectedUsingTaskInputs` | Static package/task/input graph | Global configuration and lockfile changes select all; missing Git history can fall back broadly | Yes: caller-named Turbo tasks | `--dry=json` shows planned tasks, but not an argument for every skipped task | No built-in miss classifier found | No cross-class acceptance sufficiency claim | Consume affected tasks/packages and global-change signals | +| [Jest `--findRelatedTests`](https://jestjs.io/docs/30.0/cli#--findrelatedtests-spaceseparatedlistofsourcefiles) | Supplied source files and Jest's module dependency information | Test files/tests | Static module-resolution evidence, not runtime coverage | No general external uncertainty/policy model; caller owns source list and configuration | No; tests only | Lists selected tests, not a first-class reason for each omitted test | No | “Related” test selection, not acceptance sufficiency | Use its related-test result as one test-evidence provider | +| [Vitest `related` / `--changed`](https://vitest.dev/guide/cli) | Supplied files or Git changes plus static imports | Test files | Static imports; documented dynamic-import limitation | `forceRerunTriggers` and config/package changes can force the full suite | No; tests only | No durable per-skip argument in the core CLI | No general miss observation contract | No acceptance sufficiency claim | Use related/changed output and propagate its limitations | +| [pytest-testmon](https://www.testmon.org/) | Per-test executed-code dependencies from Coverage.py plus source/block changes and persisted `.testmondata` | Pytest tests | Runtime coverage plus source analysis | First qualifying run executes all; failed tests rerun; mode conflicts can disable selection/collection | No; tests only | Selection can be inspected, but no cross-catalog reason for every skip | `--testmon-noselect` runs all while prioritizing likely failures, but no durable generic miss contract | Makes scoped affected-test claims, not whole-change verification sufficiency | Use runtime test-to-code evidence and explicit database readiness state | +| [vitest-affected](https://github.com/craigvandotcom/vitest-affected) | Git changes, cached Vitest runtime import data, delta static parsing, explicit full-suite triggers | Vitest test files | Runtime-observed imports plus static delta parsing | Cache/git/graph failure falls back to full; first run is full; documented non-import gaps need triggers; stale cache warns but does not force full | No; tests only | Yes: selected chains and a why-not explanation based on absence from the cached graph | Yes: predicts selection while the full suite runs and emits decision data | Explicitly advises retaining full/periodic truth; no cross-class sufficiency claim | Reuse runtime graph, explain trails, fallback signals, and shadow observations rather than rebuilding them | +| [Bazel query](https://bazel.build/query/quickstart) | Declared target/build graph and query universe | Build/test targets | Static declared build graph | Query scope and graph completeness are caller responsibilities; `--keep_going` tolerates errors but is not sufficiency | Queries can select any target; Bazel can build/test selected targets | `rdeps`, `somepath`, and `allpaths` can explain graph relationships | No generic selection-miss contract | No change-acceptance sufficiency claim | Consume target/reverse-dependency paths and configuration boundaries | +| [Pants changed targets](https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection) | Git changes, inferred/declared target dependencies, `changed` options | Targets supplied to any goal | Static target/dependency graph | Can include direct or transitive dependees; no general evidence-sufficiency state | Yes: selected targets can feed test, lint, package, and other goals | Target introspection can explain graph membership; no per-skip verification argument | No generic miss classifier | No whole-change sufficiency claim | Consume changed target closure and goal/task metadata | +| [Azure Pipelines Test Impact Analysis](https://learn.microsoft.com/en-us/azure/devops/pipelines/test/test-impact-analysis) | Managed-code test impact data and source changes | Automated tests | Runtime instrumentation/impact data | Unknown file types fall back to all tests; periodic full runs are configurable and recommended | No; tests only | Reporting exposes TIA outcome, not a generic per-skip argument | Periodic full runs provide validation opportunity, but no portable durable miss schema | Scoped platform TIA, not cross-class acceptance sufficiency | Reuse impacted-test result, unknown-type fallback, and full-run cadence evidence | + +## Research boundary + +Regression-test selection is a mature research field. Yoo and Harman's [survey](https://doi.org/10.1002/stvr.430) distinguishes minimization, modification-aware selection, and prioritization. Rothermel and Harrold's [safe regression-test selection work](https://doi.org/10.1145/248233.248262) uses “safe” in a formal, controlled-program sense. This project must not borrow that term for a multi-language, multi-check planner without equivalent proof. + +The prototype's defensible distinction is narrower: it combines potentially heterogeneous selector evidence with an explicit verification catalog and risk policy, then emits an acceptance-oriented argument covering tests and non-test checks, uncertainty, escalation, and every catalogued skip. The individual graph, coverage, affected-target, explanation, and shadow mechanisms are prior art. + +## Novelty ceiling + +No novelty is claimed. Future comparison must include full verification, native affected tooling, Affected Verification, and Affected Verification consuming native tooling. Value exists only if the composed argument improves workload without increasing relevant misses under controlled shadow evidence. + diff --git a/README.md b/README.md index 565c383..33e1951 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,37 @@ -# affected-verification -Minimum defensible verification planning from change-impact, coverage, policy, and risk evidence +# Affected Verification + +Affected Verification deterministically selects the smallest verification workload whose sufficiency can be defended from the available change-impact, dependency, coverage, policy, and risk evidence. + +The operative claim is **minimum defensible verification**, not mathematical global minimality. Unknown impact is never permission to skip work. + +This repository contains a dependency-free Node.js 20 prototype. It consumes normalized change, impact, verification-catalog, and policy data and emits an `opsle.affected-verification.plan.v1` argument containing selected checks, skipped checks, exact reasons, provenance hashes, uncertainty, escalation, and sufficiency. It plans work; it does not run CI. + +## Try it + +```bash +node bin/affected-verification.js \ + fixture unrelated-large-suite \ + --receipt /tmp/av-receipt.json +``` + +Canonical plan JSON is written to stdout. The `opsle.value-receipt.v1` is written only to the requested sidecar, and one `[Affected Verification]` indicator is written to stderr. The fixture reports exactly 14 of 1,043 test executions selected, 1,029 skipped, plus one lint and one typecheck; the test-execution reduction is an `EXACT` calculation, not a time, cost, token, or correctness claim. + +```bash +npm run verify +``` + +## Contract and evidence + +- [SPEC.md](SPEC.md) — normative prototype contract and sufficiency states +- [PRIOR_ART.md](PRIOR_ART.md) — source-linked reconciliation with existing selectors +- [ARCHITECTURE.md](ARCHITECTURE.md) — adapters, project boundaries, shadow mode, and trust ramp +- [BENCHMARK.md](BENCHMARK.md) — controlled research plan; no results are claimed +- [LIMITATIONS.md](LIMITATIONS.md) — current claim ceiling and non-goals +- [fixtures/scenarios.js](fixtures/scenarios.js) and [fixtures/negative-cases.js](fixtures/negative-cases.js) — twelve positive/boundary scenarios plus explicit conflicting, malformed, impossible, and tampered cases +- [schemas/plan-v1.schema.json](schemas/plan-v1.schema.json) — plan shape + +## Status + +This is a narrow research prototype, not a trusted replacement for full verification. It has not been benchmarked on a real repository, and it has no production adapters. + +Apache-2.0. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2759593 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Security + +Please report vulnerabilities through GitHub's private vulnerability reporting for `opsle/affected-verification` when available, or contact the repository maintainers privately. + +The prototype does not execute catalog commands, access a network, load plugins, or evaluate model output. Treat normalized evidence, catalogs, policies, and shadow relevance as untrusted input. A plan must not authorize execution by itself. + diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..cb822cc --- /dev/null +++ b/SPEC.md @@ -0,0 +1,97 @@ +# Affected Verification specification + +Status: normative prototype contract +Plan identity: `opsle.affected-verification.plan.v1` +Input identity: `opsle.affected-verification.input.v1` + +## 1. Canonical definition + +Affected Verification deterministically selects the smallest verification workload whose sufficiency can be defended from the available change-impact, dependency, coverage, policy, and risk evidence. + +“Smallest” is bounded by the normalized evidence and configured policy. This contract rejects the universal claim that a planner can prove the mathematically minimum number of checks needed for every program and change. + +## 2. Inputs + +One input object contains: + +- a base revision, target revision, and nonempty changed-path list, with optional changed regions and explicit risk tags; +- versioned evidence providers, a component dependency graph, path-to-component impact claims, and an explicit completeness bit; +- a complete-or-incomplete verification catalog; +- an identified, versioned policy containing deterministic matching and escalation rules. + +A verification catalog entry has a stable check ID, check type, literal command identity, component scope, tags, and a nonnegative integer count of test executions when the entry is a test check. Optional cost metadata is descriptive only in v1 and never drives selection. + +The prototype accepts normalized evidence. Git, Nx, Turbo, Jest, Vitest, testmon, coverage, CODEOWNERS, schema tools, and source-graph integrations belong in replaceable adapters that produce this format. + +## 3. Verification is broader than tests + +V1 recognizes unit, integration, end-to-end, lint, typecheck, build, compiler, schema, migration, API-contract, security, snapshot, visual, smoke, release, documentation, and test-infrastructure checks, plus an explicit `other` type. A check is an indivisible catalog entry in v1. The planner does not schedule or execute it. + +## 4. Deterministic planning + +For valid input, the reference planner: + +1. maps known changed paths to directly affected components; +2. walks reverse dependencies to a fixed point; +3. matches identified policy rules; +4. selects checks whose declared scope intersects the affected closure; +5. adds all checks carrying tags required by matched policy rules and refuses sufficiency if any required tag has no catalogued check; +6. broadens to the complete catalog when impact is unknown or incomplete; +7. classifies every remaining catalog check as skipped with an evidence-backed reason; +8. hashes canonical input and canonical plan content with SHA-256. + +Inputs, checks, providers, rules, components, dependencies, selections, skips, and reasons are sorted where order is not semantic. The same input yields byte-equivalent canonical output and the same plan identity. + +V1 does not solve weighted set cover and does not choose among alternative commands. If a future catalog expresses equally sufficient alternatives, its policy must define a deterministic tie-break or the planner must report ambiguity. It must not silently use ambient timing or model judgment. + +## 5. Sufficiency and escalation states + +`SUFFICIENT_TARGETED` +: Complete impact and catalog evidence supports the targeted selection and every affected component has catalog coverage. + +`SUFFICIENT_BROADENED` +: The targeted closure is complete, but matched risk or policy rules add verification beyond direct scope. + +`FULL_VERIFICATION_REQUIRED` +: A matched policy requires all catalogued checks, or incomplete/unknown impact makes targeted sufficiency indefensible and the catalog is declared complete. + +`INSUFFICIENT_EVIDENCE` +: Even selecting every known check cannot establish sufficiency, including when the verification catalog is incomplete or an affected component has no catalog coverage. + +`INVALID_INPUT` +: The input is malformed, contradictory, numerically impossible, references unknown entities, duplicates identity-bearing evidence, or is invalidated by policy. CLI errors use `opsle.affected-verification.error.v1`, exit 2, and emit no success indicator or value receipt. Policy invalidation uses the more specific code `PLAN_INVALIDATED` while retaining the `INVALID_INPUT` semantics. + +The invariant is `UNKNOWN != SAFE TO SKIP`. Unknown or incomplete impact selects the full known catalog. If that catalog is incomplete, the result remains `INSUFFICIENT_EVIDENCE` rather than pretending the known full set is sufficient. + +## 6. Selection and skip arguments + +Every selected check contains one or more reason records with a stable code, exact detail, and evidence references. Every skipped check must do the same. V1 emits `OUTSIDE_TRANSITIVE_IMPACT_SET` only when the scope misses the complete affected closure and no matched policy requires the check. + +“Not selected” is not a reason. An implementation must fail conformance if a skipped check has no reason. + +## 7. Plan contract + +The plan contains: + +- schema and deterministic plan identity; +- base, target, change identity, paths, and supported changed regions; +- input, evidence, catalog, policy, and provider identities; +- direct and reverse-dependent affected components; +- selected and skipped checks with rationale; +- risk tags and matched policy rules; +- uncertainty and escalation states; +- sufficiency classification and the bounded claim. + +The exact structural profile is [schemas/plan-v1.schema.json](schemas/plan-v1.schema.json). `plan_identity` is SHA-256 over canonical plan content with that field omitted. `change.identity` is caller supplied or deterministically derived. Provider identities are recorded, not independently attested. + +## 8. Visible Value + +The planner can emit an `opsle.value-receipt.v1` sidecar. Exact measurements are catalog checks available, selected, and skipped; declared test executions available, selected, and skipped; and the rational test-execution reduction. Unlike verification types are not collapsed into a workload percentage. Counts are planning output, not observed execution. + +The receipt explicitly makes no time, token, cost, correctness, or causal savings claim. Canonical plan JSON stays on stdout and the named operator indicator stays on stderr. + +## 9. Failure behavior and compatibility + +Unknown fields are rejected in the normalized input. Duplicate provider, component, impact, check, rule, executed-check, or failure identities are rejected. Unknown graph edges, scopes, and shadow checks are rejected. Negative, noninteger, unsafe-integer, nonfinite, or semantically incompatible numeric values are rejected. Nonempty matcher dimensions within one policy rule compose conjunctively; values inside one dimension are alternatives. + +V1 compatibility is exact by schema identity. Additive or semantic changes require a new schema version or a documented compatible profile. No external package or repository is a runtime dependency. diff --git a/bin/affected-verification.js b/bin/affected-verification.js new file mode 100755 index 0000000..b53bd64 --- /dev/null +++ b/bin/affected-verification.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node +import { readFileSync, writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { + InputError, + buildValueReceipt, + canonicalJson, + classifyShadow, + operatorIndicator, + planVerification, +} from '../src/index.js'; +import { scenarios } from '../fixtures/scenarios.js'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function parseReceipt(args) { + const index = args.indexOf('--receipt'); + if (index === -1) return null; + if (!args[index + 1]) throw new InputError(['--receipt requires a path']); + return args[index + 1]; +} + +function loadFixture(id) { + const fixture = scenarios.find((item) => item.id === id); + if (!fixture) throw new InputError([`unknown fixture ${id}`]); + return fixture.input; +} + +function usage() { + return [ + 'affected-verification plan [--receipt ]', + 'affected-verification fixture [--receipt ]', + 'affected-verification shadow ', + ].join('\n'); +} + +try { + const [command, ...args] = process.argv.slice(2); + if (command === '--help' || command === '-h' || !command) { + process.stdout.write(`${usage()}\n`); + } else if (command === 'plan' || command === 'fixture') { + if (!args[0]) throw new InputError([`${command} requires an input`]); + const input = command === 'plan' ? readJson(args[0]) : loadFixture(args[0]); + const plan = planVerification(input); + const receiptPath = parseReceipt(args); + if (receiptPath) writeFileSync(receiptPath, `${canonicalJson(buildValueReceipt(plan))}\n`); + process.stdout.write(`${canonicalJson(plan)}\n`); + process.stderr.write(`${operatorIndicator(plan)}\n`); + } else if (command === 'shadow') { + if (!args[0] || !args[1]) throw new InputError(['shadow requires a plan and full-run input']); + process.stdout.write(`${canonicalJson(classifyShadow(readJson(args[0]), readJson(args[1])))}\n`); + } else { + throw new InputError([`unknown command ${command}`]); + } +} catch (error) { + if (error instanceof InputError || error instanceof SyntaxError) { + const packet = { + schema: 'opsle.affected-verification.error.v1', + code: error.code ?? 'INVALID_JSON', + issues: error.issues ?? [error.message], + }; + process.stdout.write(`${canonicalJson(packet)}\n`); + process.exitCode = 2; + } else { + throw error; + } +} diff --git a/fixtures/negative-cases.js b/fixtures/negative-cases.js new file mode 100644 index 0000000..a710a4c --- /dev/null +++ b/fixtures/negative-cases.js @@ -0,0 +1,32 @@ +import { fixture } from './scenarios.js'; + +function conflictingImpact() { + const input = fixture('isolated-implementation'); + input.evidence.impacts.push({ ...input.evidence.impacts[0], components: ['unrelated'] }); + return input; +} + +function impossibleTestCount() { + const input = fixture('isolated-implementation'); + input.catalog.checks[0].test_executions = -1; + return input; +} + +function unknownDependency() { + const input = fixture('isolated-implementation'); + input.evidence.components[0].dependencies.push('missing-component'); + return input; +} + +export const negativeCases = [ + { id: 'duplicate-conflicting-impact', expected_error: 'INVALID_INPUT', input: conflictingImpact() }, + { id: 'impossible-negative-test-count', expected_error: 'INVALID_INPUT', input: impossibleTestCount() }, + { id: 'unknown-component-dependency', expected_error: 'INVALID_INPUT', input: unknownDependency() }, +]; + +export function tamperPlan(plan) { + const tampered = structuredClone(plan); + tampered.selected_checks[0].command = 'changed-after-planning'; + return tampered; +} + diff --git a/fixtures/scenarios.js b/fixtures/scenarios.js new file mode 100644 index 0000000..38e0aa7 --- /dev/null +++ b/fixtures/scenarios.js @@ -0,0 +1,164 @@ +const components = [ + { id: 'docs', dependencies: [] }, + { id: 'isolated', dependencies: [] }, + { id: 'isolated-tests', dependencies: [] }, + { id: 'feature', dependencies: [] }, + { id: 'consumer', dependencies: ['feature'] }, + { id: 'shared', dependencies: [] }, + { id: 'subsystem-a', dependencies: ['shared'] }, + { id: 'subsystem-b', dependencies: ['shared'] }, + { id: 'global-config', dependencies: [] }, + { id: 'auth', dependencies: ['shared'] }, + { id: 'database', dependencies: [] }, + { id: 'test-infrastructure', dependencies: [] }, + { id: 'unrelated', dependencies: [] }, +]; + +const codeComponents = components + .map((item) => item.id) + .filter((id) => !['docs', 'global-config', 'unrelated'].includes(id)); + +const checks = [ + { id: 'docs.validate', type: 'documentation', command: 'node tools/docs-check.js', scope: { components: ['docs'] }, tags: ['documentation'], test_executions: 0 }, + { id: 'config.validate', type: 'build', command: 'node tools/config-check.js', scope: { components: ['global-config'] }, tags: ['global'], test_executions: 0 }, + { id: 'isolated.unit', type: 'unit-test', command: 'node --test tests/isolated.test.js', scope: { components: ['isolated'] }, tags: ['direct'], test_executions: 14 }, + { id: 'isolated.test-file', type: 'unit-test', command: 'node --test tests/isolated.test.js', scope: { components: ['isolated-tests'] }, tags: ['test-change'], test_executions: 1 }, + { id: 'feature.unit', type: 'unit-test', command: 'node --test tests/feature.test.js', scope: { components: ['feature'] }, tags: ['direct'], test_executions: 6 }, + { id: 'consumer.integration', type: 'integration-test', command: 'node --test tests/consumer.test.js', scope: { components: ['consumer'] }, tags: ['dependent'], test_executions: 4 }, + { id: 'shared.unit', type: 'unit-test', command: 'node --test tests/shared.test.js', scope: { components: ['shared'] }, tags: ['shared'], test_executions: 10 }, + { id: 'subsystem-a.integration', type: 'integration-test', command: 'node --test tests/subsystem-a.test.js', scope: { components: ['subsystem-a'] }, tags: ['subsystem'], test_executions: 8 }, + { id: 'subsystem-b.integration', type: 'integration-test', command: 'node --test tests/subsystem-b.test.js', scope: { components: ['subsystem-b'] }, tags: ['subsystem'], test_executions: 8 }, + { id: 'auth.unit', type: 'unit-test', command: 'node --test tests/auth.test.js', scope: { components: ['auth'] }, tags: ['security-boundary'], test_executions: 5 }, + { id: 'auth.security', type: 'security', command: 'node tools/security-check.js', scope: { components: ['auth'] }, tags: ['security-boundary'], test_executions: 0 }, + { id: 'database.schema', type: 'schema', command: 'node tools/schema-check.js', scope: { components: ['database'] }, tags: ['migration'], test_executions: 0 }, + { id: 'database.migration', type: 'migration', command: 'node tools/migration-check.js', scope: { components: ['database'] }, tags: ['migration'], test_executions: 0 }, + { id: 'database.integration', type: 'integration-test', command: 'node --test tests/migration.test.js', scope: { components: ['database'] }, tags: ['migration'], test_executions: 7 }, + { id: 'test-infrastructure.self', type: 'test-infrastructure', command: 'node --test tests/harness.test.js', scope: { components: ['test-infrastructure'] }, tags: ['test-infrastructure'], test_executions: 0 }, + { id: 'project.typecheck', type: 'typecheck', command: 'node tools/typecheck.js', scope: { components: codeComponents }, tags: ['code-quality'], test_executions: 0 }, + { id: 'project.lint', type: 'lint', command: 'node tools/lint.js', scope: { components: codeComponents }, tags: ['code-quality'], test_executions: 0 }, + { id: 'release.smoke', type: 'smoke', command: 'node tools/smoke.js', scope: { components: ['consumer', 'auth', 'database'] }, tags: ['release'], test_executions: 0 }, + { id: 'unrelated.large-suite', type: 'end-to-end-test', command: 'node --test tests/unrelated/**/*.test.js', scope: { components: ['unrelated'] }, tags: ['unrelated'], test_executions: 1029 }, +]; + +const rules = [ + { id: 'global-configuration', match: { path_globs: ['config/**'], risk_tags: [], component_ids: [] }, escalation: 'FULL', required_check_tags: [] }, + { id: 'security-boundary', match: { path_globs: [], risk_tags: ['security-boundary'], component_ids: [] }, escalation: 'BROADEN', required_check_tags: ['security-boundary'] }, + { id: 'database-migration', match: { path_globs: [], risk_tags: ['migration'], component_ids: [] }, escalation: 'BROADEN', required_check_tags: ['migration'] }, + { id: 'shared-boundary', match: { path_globs: [], risk_tags: [], component_ids: ['shared'] }, escalation: 'BROADEN', required_check_tags: ['subsystem'] }, + { id: 'verification-metadata', match: { path_globs: ['verification/**'], risk_tags: [], component_ids: [] }, escalation: 'INVALIDATE', required_check_tags: [] }, +]; + +function input({ path, component, riskTags = [], complete = true, confidence = 'KNOWN', target }) { + return { + schema: 'opsle.affected-verification.input.v1', + change: { + base_revision: 'base000000000000000000000000000000000000', + target_revision: target, + paths: [{ path, regions: [], risk_tags: riskTags }], + }, + evidence: { + identity: `evidence:${target}`, + complete, + providers: [{ id: 'synthetic-graph', kind: 'NORMALIZED_FIXTURE', version: '1.0.0', identity: 'sha256:fixture-provider' }], + components, + impacts: [{ path, components: component ? [component] : [], confidence, reason: confidence === 'KNOWN' ? 'Synthetic path ownership' : 'No defensible owner mapping' }], + }, + catalog: { identity: 'catalog:synthetic-v1', complete: true, checks }, + policy: { identity: 'policy:synthetic-v1', version: '1.0.0', rules }, + }; +} + +const largeSkipInput = input({ + path: 'src/isolated-large.js', + component: 'isolated', + target: 'target-large-skip', +}); +largeSkipInput.catalog = { + identity: 'catalog:synthetic-large-suite-v1', + complete: true, + checks: checks.filter((check) => [ + 'isolated.unit', + 'project.lint', + 'project.typecheck', + 'unrelated.large-suite', + ].includes(check.id)), +}; + +export const scenarios = [ + { + id: 'readme-only', + description: 'README-only change selects documentation validation and no code regression suite.', + expected_sufficiency: 'SUFFICIENT_TARGETED', + input: input({ path: 'README.md', component: 'docs', target: 'target-readme' }), + }, + { + id: 'isolated-implementation', + description: 'Isolated implementation change selects direct tests and code checks.', + expected_sufficiency: 'SUFFICIENT_TARGETED', + input: input({ path: 'src/isolated.js', component: 'isolated', target: 'target-isolated' }), + }, + { + id: 'reverse-dependent', + description: 'Feature change selects feature and reverse-dependent consumer verification.', + expected_sufficiency: 'SUFFICIENT_TARGETED', + input: input({ path: 'src/feature.js', component: 'feature', target: 'target-dependent' }), + }, + { + id: 'shared-common', + description: 'Shared module change broadens through multiple dependent subsystems.', + expected_sufficiency: 'SUFFICIENT_BROADENED', + input: input({ path: 'src/shared.js', component: 'shared', target: 'target-shared' }), + }, + { + id: 'global-configuration', + description: 'Global configuration policy requires the full catalog.', + expected_sufficiency: 'FULL_VERIFICATION_REQUIRED', + input: input({ path: 'config/runtime.json', component: 'global-config', riskTags: ['global'], target: 'target-config' }), + }, + { + id: 'authentication-boundary', + description: 'Authentication change triggers security-policy broadening.', + expected_sufficiency: 'SUFFICIENT_BROADENED', + input: input({ path: 'src/auth/session.js', component: 'auth', riskTags: ['security-boundary'], target: 'target-auth' }), + }, + { + id: 'database-migration', + description: 'Migration change selects schema, migration, and integration checks.', + expected_sufficiency: 'SUFFICIENT_BROADENED', + input: input({ path: 'migrations/0042.sql', component: 'database', riskTags: ['migration'], target: 'target-db' }), + }, + { + id: 'incomplete-dependency-graph', + description: 'Incomplete graph cannot claim targeted sufficiency and fails closed to the full catalog.', + expected_sufficiency: 'FULL_VERIFICATION_REQUIRED', + input: input({ path: 'src/isolated.js', component: 'isolated', complete: false, target: 'target-incomplete' }), + }, + { + id: 'unknown-changed-file', + description: 'Unknown path ownership conservatively requires the full catalog.', + expected_sufficiency: 'FULL_VERIFICATION_REQUIRED', + input: input({ path: 'mystery/generated.xyz', component: null, confidence: 'UNKNOWN', target: 'target-unknown' }), + }, + { + id: 'test-file-only', + description: 'Test-only change selects the corresponding test-specific check.', + expected_sufficiency: 'SUFFICIENT_TARGETED', + input: input({ path: 'tests/isolated.test.js', component: 'isolated-tests', riskTags: ['test-only'], target: 'target-test' }), + }, + { + id: 'unrelated-large-suite', + description: 'Fourteen related test executions are selected while 1,029 unrelated executions are defensibly skipped.', + expected_sufficiency: 'SUFFICIENT_TARGETED', + input: largeSkipInput, + }, + { + id: 'critical-verification-metadata', + description: 'Changing the catalog invalidates a plan based on that catalog.', + expected_error: 'PLAN_INVALIDATED', + input: input({ path: 'verification/catalog.json', component: 'test-infrastructure', target: 'target-metadata' }), + }, +]; + +export function fixture(id) { + return structuredClone(scenarios.find((item) => item.id === id)?.input); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c1ced78 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "@opsle/affected-verification", + "version": "0.1.0", + "private": true, + "type": "module", + "bin": { + "affected-verification": "./bin/affected-verification.js" + }, + "exports": "./src/index.js", + "scripts": { + "check": "node --check src/*.js && node --check bin/*.js && node --check tools/*.js && node --check tests/*.js", + "test": "node --test", + "conformance": "node ./tools/conformance.js", + "determinism": "node --test --test-name-pattern='deterministic|stable|identical'", + "verify": "npm run check && npm test && npm run conformance && npm run determinism" + }, + "engines": { + "node": ">=20" + }, + "license": "Apache-2.0" +} + diff --git a/schemas/plan-v1.schema.json b/schemas/plan-v1.schema.json new file mode 100644 index 0000000..c8d9aed --- /dev/null +++ b/schemas/plan-v1.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/opsle/affected-verification/schemas/plan-v1.schema.json", + "title": "Affected Verification plan v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "plan_identity", "change", "provenance", "affected_components", "selected_checks", "skipped_checks", "risk", "uncertainty", "escalation", "sufficiency", "argument"], + "properties": { + "schema": { "const": "opsle.affected-verification.plan.v1" }, + "plan_identity": { "$ref": "#/$defs/identity" }, + "change": { + "type": "object", + "additionalProperties": false, + "required": ["base_revision", "target_revision", "identity", "changed_paths", "changed_regions"], + "properties": { + "base_revision": { "type": "string", "minLength": 1 }, + "target_revision": { "type": "string", "minLength": 1 }, + "identity": { "type": "string", "minLength": 1 }, + "changed_paths": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "changed_regions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "region"], + "properties": { "path": { "type": "string", "minLength": 1 }, "region": { "type": "string", "minLength": 1 } } + } + } + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["input_hash", "evidence_identity", "verification_catalog_identity", "policy_identity", "policy_version", "providers"], + "properties": { + "input_hash": { "$ref": "#/$defs/identity" }, + "evidence_identity": { "type": "string", "minLength": 1 }, + "verification_catalog_identity": { "type": "string", "minLength": 1 }, + "policy_identity": { "type": "string", "minLength": 1 }, + "policy_version": { "type": "string", "minLength": 1 }, + "providers": { "type": "array", "minItems": 1, "items": { "type": "object" } } + } + }, + "affected_components": { "type": "array", "items": { "$ref": "#/$defs/component" } }, + "selected_checks": { "type": "array", "items": { "$ref": "#/$defs/check" } }, + "skipped_checks": { "type": "array", "items": { "$ref": "#/$defs/check" } }, + "risk": { "type": "object", "required": ["classification", "tags", "matched_policy_rules"], "properties": { "classification": { "enum": ["BASELINE", "DECLARED_TAGGED", "POLICY_BROADENED", "POLICY_FULL"] }, "tags": { "type": "array" }, "matched_policy_rules": { "type": "array" } } }, + "uncertainty": { "type": "object", "required": ["state", "reasons"], "properties": { "state": { "enum": ["NONE", "PRESENT"] }, "reasons": { "type": "array" } } }, + "escalation": { "type": "object", "required": ["state", "reasons"], "properties": { "state": { "enum": ["NONE", "BROADENED", "FULL"] }, "reasons": { "type": "array" } } }, + "sufficiency": { "enum": ["SUFFICIENT_TARGETED", "SUFFICIENT_BROADENED", "FULL_VERIFICATION_REQUIRED", "INSUFFICIENT_EVIDENCE"] }, + "argument": { + "type": "object", + "additionalProperties": false, + "required": ["claim", "unknown_is_safe_to_skip", "every_skip_explained"], + "properties": { + "claim": { "type": "string", "minLength": 1 }, + "unknown_is_safe_to_skip": { "const": false }, + "every_skip_explained": { "const": true } + } + } + }, + "$defs": { + "identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "component": { + "type": "object", + "additionalProperties": false, + "required": ["id", "direct", "via_dependencies"], + "properties": { "id": { "type": "string", "minLength": 1 }, "direct": { "type": "boolean" }, "via_dependencies": { "type": "array", "items": { "type": "string" } } } + }, + "reason": { + "type": "object", + "additionalProperties": false, + "required": ["code", "detail", "evidence_refs"], + "properties": { "code": { "type": "string", "minLength": 1 }, "detail": { "type": "string", "minLength": 1 }, "evidence_refs": { "type": "array", "minItems": 1, "items": { "type": "string" } } } + }, + "check": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "command", "scope", "tags", "test_executions", "reasons"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "type": "string", "minLength": 1 }, + "command": { "type": "string", "minLength": 1 }, + "scope": { "type": "object", "required": ["components"], "properties": { "components": { "type": "array", "minItems": 1 } } }, + "tags": { "type": "array" }, + "test_executions": { "type": "integer", "minimum": 0 }, + "reasons": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/reason" } } + } + } + } +} diff --git a/src/canonical.js b/src/canonical.js new file mode 100644 index 0000000..6ca7476 --- /dev/null +++ b/src/canonical.js @@ -0,0 +1,35 @@ +import { createHash } from 'node:crypto'; + +function normalize(value) { + if (Array.isArray(value)) { + return value.map(normalize); + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => value[key] !== undefined) + .map((key) => [key, normalize(value[key])]), + ); + } + return value; +} + +export function canonicalJson(value) { + return JSON.stringify(normalize(value)); +} + +export function contentIdentity(value) { + return `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`; +} + +export function deepFreeze(value) { + if (value && typeof value === 'object' && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) { + deepFreeze(child); + } + } + return value; +} + diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..31725b1 --- /dev/null +++ b/src/index.js @@ -0,0 +1,12 @@ +export { canonicalJson, contentIdentity } from './canonical.js'; +export { planVerification } from './planner.js'; +export { classifyShadow } from './shadow.js'; +export { buildValueReceipt, operatorIndicator } from './value-receipt.js'; +export { + INPUT_SCHEMA, + PLAN_SCHEMA, + SHADOW_INPUT_SCHEMA, + SHADOW_OBSERVATION_SCHEMA, + InputError, +} from './validate.js'; + diff --git a/src/planner.js b/src/planner.js new file mode 100644 index 0000000..76acd31 --- /dev/null +++ b/src/planner.js @@ -0,0 +1,264 @@ +import { canonicalJson, contentIdentity, deepFreeze } from './canonical.js'; +import { InputError, PLAN_SCHEMA, validateInput } from './validate.js'; + +const ESCALATION_ORDER = { NONE: 0, BROADEN: 1, FULL: 2, INVALIDATE: 3 }; + +function globMatches(pattern, path) { + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*\*/g, '\u0000') + .replace(/\*/g, '[^/]*') + .replace(/\u0000/g, '.*') + .replace(/\?/g, '[^/]'); + return new RegExp(`^${escaped}$`).test(path); +} + +function sortedUnique(values) { + return [...new Set(values)].sort(); +} + +function reason(code, detail, evidenceRefs) { + return { code, detail, evidence_refs: sortedUnique(evidenceRefs) }; +} + +function directComponents(input) { + return sortedUnique(input.evidence.impacts + .filter((impact) => impact.confidence === 'KNOWN') + .flatMap((impact) => impact.components)); +} + +function reverseClosure(components, direct) { + const affected = new Set(direct); + const via = new Map(direct.map((id) => [id, []])); + let changed = true; + while (changed) { + changed = false; + for (const component of components) { + if (affected.has(component.id)) continue; + const triggering = component.dependencies.filter((dependency) => affected.has(dependency)).sort(); + if (triggering.length) { + affected.add(component.id); + via.set(component.id, triggering); + changed = true; + } + } + } + return [...affected].sort().map((id) => ({ + id, + direct: direct.includes(id), + via_dependencies: via.get(id) ?? [], + })); +} + +function matchedRules(input, affectedIds) { + const changedPaths = input.change.paths.map((item) => item.path); + const riskTags = new Set(input.change.paths.flatMap((item) => item.risk_tags ?? [])); + return input.policy.rules.filter((rule) => { + const groups = [ + [rule.match.path_globs ?? [], (glob) => changedPaths.some((path) => globMatches(glob, path))], + [rule.match.risk_tags ?? [], (tag) => riskTags.has(tag)], + [rule.match.component_ids ?? [], (id) => affectedIds.has(id)], + ]; + return groups.every(([values, matches]) => values.length === 0 || values.some(matches)); + }).sort((a, b) => a.id.localeCompare(b.id)); +} + +function selectionReason(check, affectedIds, requiredBy) { + const intersection = check.scope.components.filter((component) => component === '*' || affectedIds.has(component)).sort(); + const reasons = []; + if (intersection.length) { + reasons.push(reason( + 'SCOPE_INTERSECTS_AFFECTED_COMPONENT', + `Check scope intersects: ${intersection.join(', ')}`, + ['impact-evidence', 'verification-catalog'], + )); + } + for (const rule of requiredBy) { + reasons.push(reason( + 'REQUIRED_BY_POLICY', + `Policy rule ${rule.id} requires one of this check's tags`, + [`policy-rule:${rule.id}`, 'verification-catalog'], + )); + } + return reasons; +} + +function mapCheck(check, reasons) { + return { + id: check.id, + type: check.type, + command: check.command, + scope: { components: [...check.scope.components].sort() }, + tags: [...check.tags].sort(), + test_executions: check.test_executions, + reasons, + }; +} + +function uncertainty(input, affected) { + const reasons = []; + const impacts = new Map(input.evidence.impacts.map((item) => [item.path, item])); + if (!input.evidence.complete) reasons.push('DEPENDENCY_EVIDENCE_INCOMPLETE'); + if (!input.catalog.complete) reasons.push('VERIFICATION_CATALOG_INCOMPLETE'); + for (const changed of input.change.paths) { + const impact = impacts.get(changed.path); + if (!impact) reasons.push(`NO_IMPACT_EVIDENCE:${changed.path}`); + else if (impact.confidence === 'UNKNOWN') reasons.push(`UNKNOWN_IMPACT:${changed.path}`); + } + const covered = new Set(); + for (const check of input.catalog.checks) { + for (const component of check.scope.components) covered.add(component); + } + if (!covered.has('*')) { + for (const item of affected) { + if (!covered.has(item.id)) reasons.push(`NO_VERIFICATION_COVERAGE:${item.id}`); + } + } + return sortedUnique(reasons); +} + +export function planVerification(rawInput) { + const input = validateInput(rawInput); + const inputHash = contentIdentity(input); + const changeIdentity = input.change.identity ?? contentIdentity({ + base_revision: input.change.base_revision, + paths: input.change.paths, + target_revision: input.change.target_revision, + }); + const direct = directComponents(input); + const affected = reverseClosure(input.evidence.components, direct); + const affectedIds = new Set(affected.map((item) => item.id)); + const policies = matchedRules(input, affectedIds); + const maxEscalation = policies.reduce( + (current, rule) => ESCALATION_ORDER[rule.escalation] > ESCALATION_ORDER[current] + ? rule.escalation + : current, + 'NONE', + ); + if (maxEscalation === 'INVALIDATE') { + throw new InputError( + policies.filter((rule) => rule.escalation === 'INVALIDATE').map((rule) => `policy rule ${rule.id} invalidates planning`), + 'PLAN_INVALIDATED', + ); + } + + let uncertaintyReasons = uncertainty(input, affected); + for (const rule of policies) { + for (const tag of rule.required_check_tags) { + if (!input.catalog.checks.some((check) => check.tags.includes(tag))) { + uncertaintyReasons.push(`POLICY_REQUIREMENT_UNSATISFIED:${rule.id}:${tag}`); + } + } + } + uncertaintyReasons = sortedUnique(uncertaintyReasons); + const requiredTagsByRule = policies.map((rule) => ({ + ...rule, + tags: new Set(rule.required_check_tags), + })); + let selectAll = maxEscalation === 'FULL'; + let sufficiency = maxEscalation === 'BROADEN' + ? 'SUFFICIENT_BROADENED' + : 'SUFFICIENT_TARGETED'; + if (uncertaintyReasons.length) { + selectAll = true; + const cannotBeRepairedByFullSelection = uncertaintyReasons.some((item) => + item.startsWith('NO_VERIFICATION_COVERAGE:') + || item.startsWith('POLICY_REQUIREMENT_UNSATISFIED:'), + ); + sufficiency = input.catalog.complete && !cannotBeRepairedByFullSelection + ? 'FULL_VERIFICATION_REQUIRED' + : 'INSUFFICIENT_EVIDENCE'; + } else if (selectAll) { + sufficiency = 'FULL_VERIFICATION_REQUIRED'; + } + + const selected = []; + const skipped = []; + for (const check of [...input.catalog.checks].sort((a, b) => a.id.localeCompare(b.id))) { + const requiredBy = requiredTagsByRule.filter((rule) => check.tags.some((tag) => rule.tags.has(tag))); + const scopeIntersects = check.scope.components.some((component) => component === '*' || affectedIds.has(component)); + if (selectAll || scopeIntersects || requiredBy.length) { + const reasons = selectionReason(check, affectedIds, requiredBy); + if (selectAll) { + reasons.push(reason( + uncertaintyReasons.length ? 'FAIL_CLOSED_FULL_SELECTION' : 'POLICY_REQUIRES_FULL_SELECTION', + uncertaintyReasons.length + ? 'All catalogued checks selected because targeted sufficiency cannot be defended' + : 'All catalogued checks selected by matched policy', + uncertaintyReasons.length ? ['impact-evidence', 'verification-catalog'] : policies.map((rule) => `policy-rule:${rule.id}`), + )); + } + selected.push(mapCheck(check, reasons)); + } else { + skipped.push(mapCheck(check, [reason( + 'OUTSIDE_TRANSITIVE_IMPACT_SET', + 'Check scope does not intersect the direct or reverse-dependent impact set and no matched policy requires it', + ['impact-evidence', 'verification-catalog', 'verification-policy'], + )])); + } + } + + const escalationState = selectAll + ? 'FULL' + : maxEscalation === 'BROADEN' + ? 'BROADENED' + : 'NONE'; + const planWithoutIdentity = { + schema: PLAN_SCHEMA, + plan_identity: null, + change: { + base_revision: input.change.base_revision, + target_revision: input.change.target_revision, + identity: changeIdentity, + changed_paths: input.change.paths.map((item) => item.path).sort(), + changed_regions: input.change.paths + .flatMap((item) => item.regions.map((region) => ({ path: item.path, region }))) + .sort((a, b) => canonicalJson(a).localeCompare(canonicalJson(b))), + }, + provenance: { + input_hash: inputHash, + evidence_identity: input.evidence.identity, + verification_catalog_identity: input.catalog.identity, + policy_identity: input.policy.identity, + policy_version: input.policy.version, + providers: [...input.evidence.providers] + .map((provider) => ({ ...provider })) + .sort((a, b) => a.id.localeCompare(b.id)), + }, + affected_components: affected, + selected_checks: selected, + skipped_checks: skipped, + risk: { + classification: maxEscalation === 'FULL' + ? 'POLICY_FULL' + : maxEscalation === 'BROADEN' + ? 'POLICY_BROADENED' + : input.change.paths.some((item) => item.risk_tags.length) + ? 'DECLARED_TAGGED' + : 'BASELINE', + tags: sortedUnique(input.change.paths.flatMap((item) => item.risk_tags)), + matched_policy_rules: policies.map((rule) => rule.id), + }, + uncertainty: { + state: uncertaintyReasons.length ? 'PRESENT' : 'NONE', + reasons: uncertaintyReasons, + }, + escalation: { + state: escalationState, + reasons: policies.filter((rule) => rule.escalation !== 'NONE').map((rule) => ({ + rule_id: rule.id, + escalation: rule.escalation, + })), + }, + sufficiency, + argument: { + claim: sufficiency === 'SUFFICIENT_TARGETED' || sufficiency === 'SUFFICIENT_BROADENED' + ? 'Smallest set justified by the normalized evidence model and configured policy; no global mathematical minimality claim.' + : 'Targeted sufficiency is not claimed.', + unknown_is_safe_to_skip: false, + every_skip_explained: skipped.every((check) => check.reasons.length > 0), + }, + }; + const planIdentity = contentIdentity({ ...planWithoutIdentity, plan_identity: undefined }); + return deepFreeze({ ...planWithoutIdentity, plan_identity: planIdentity }); +} diff --git a/src/shadow.js b/src/shadow.js new file mode 100644 index 0000000..c18f40c --- /dev/null +++ b/src/shadow.js @@ -0,0 +1,72 @@ +import { contentIdentity, deepFreeze } from './canonical.js'; +import { InputError, PLAN_SCHEMA, SHADOW_OBSERVATION_SCHEMA, validateShadowInput } from './validate.js'; + +function validatePlanIntegrity(plan) { + const issues = []; + if (!plan || typeof plan !== 'object' || Array.isArray(plan)) { + throw new InputError(['plan must be an object'], 'TAMPERED_PLAN'); + } + if (plan.schema !== PLAN_SCHEMA) issues.push(`plan.schema must be ${PLAN_SCHEMA}`); + if (!plan.change || typeof plan.change.identity !== 'string') issues.push('plan.change.identity must be a string'); + if (!plan.provenance || typeof plan.provenance.policy_identity !== 'string' || typeof plan.provenance.policy_version !== 'string') { + issues.push('plan provenance policy identity and version are required'); + } + if (!Array.isArray(plan.selected_checks) || !Array.isArray(plan.skipped_checks)) { + issues.push('plan selected_checks and skipped_checks must be arrays'); + } else { + const ids = []; + for (const [kind, checks] of [['selected', plan.selected_checks], ['skipped', plan.skipped_checks]]) { + for (const check of checks) { + if (!check || typeof check.id !== 'string') issues.push(`${kind} check id must be a string`); + else ids.push(check.id); + if (!Array.isArray(check?.reasons) || check.reasons.length === 0) issues.push(`${kind} check ${check?.id ?? ''} must have reasons`); + } + } + if (new Set(ids).size !== ids.length) issues.push('plan check ids must be unique across selected and skipped sets'); + } + const expectedIdentity = contentIdentity({ ...plan, plan_identity: undefined }); + if (plan.plan_identity !== expectedIdentity) issues.push('plan identity does not match canonical content'); + if (issues.length) throw new InputError(issues, 'TAMPERED_PLAN'); +} + +export function classifyShadow(plan, rawShadow) { + validatePlanIntegrity(plan); + const shadow = validateShadowInput(rawShadow, plan); + const selected = new Set(plan.selected_checks.map((item) => item.id)); + const skipped = new Set(plan.skipped_checks.map((item) => item.id)); + const known = new Set([...selected, ...skipped]); + const executed = new Set(shadow.executed_check_ids); + const fullRunComplete = [...known].every((id) => executed.has(id)); + const misses = shadow.failures + .filter((failure) => skipped.has(failure.check_id) && failure.relevant) + .map((failure) => ({ + check_id: failure.check_id, + exact_miss_reason: failure.reason, + })) + .sort((a, b) => a.check_id.localeCompare(b.check_id)); + const classification = misses.length + ? 'SELECTION_MISS' + : fullRunComplete + ? 'NO_SELECTION_MISS' + : 'INDETERMINATE_FULL_RUN_INCOMPLETE'; + const withoutIdentity = { + schema: SHADOW_OBSERVATION_SCHEMA, + observation_identity: null, + plan_identity: plan.plan_identity, + planner_schema: plan.schema, + policy_identity: plan.provenance.policy_identity, + policy_version: plan.provenance.policy_version, + change_identity: plan.change.identity, + predicted_selected_check_ids: [...selected].sort(), + predicted_skipped_check_ids: [...skipped].sort(), + full_run_executed_check_ids: [...executed].sort(), + full_run_failures: [...shadow.failures].sort((a, b) => a.check_id.localeCompare(b.check_id)), + selection_misses: misses, + full_run_complete: fullRunComplete, + classification, + }; + return deepFreeze({ + ...withoutIdentity, + observation_identity: contentIdentity({ ...withoutIdentity, observation_identity: undefined }), + }); +} diff --git a/src/validate.js b/src/validate.js new file mode 100644 index 0000000..577dd75 --- /dev/null +++ b/src/validate.js @@ -0,0 +1,267 @@ +export const INPUT_SCHEMA = 'opsle.affected-verification.input.v1'; +export const PLAN_SCHEMA = 'opsle.affected-verification.plan.v1'; +export const SHADOW_INPUT_SCHEMA = 'opsle.affected-verification.shadow-input.v1'; +export const SHADOW_OBSERVATION_SCHEMA = 'opsle.affected-verification.shadow-observation.v1'; + +const CHECK_TYPES = new Set([ + 'unit-test', + 'integration-test', + 'end-to-end-test', + 'lint', + 'typecheck', + 'build', + 'compiler', + 'schema', + 'migration', + 'api-contract', + 'security', + 'snapshot', + 'visual', + 'smoke', + 'release', + 'documentation', + 'test-infrastructure', + 'other', +]); +const TEST_CHECK_TYPES = new Set(['unit-test', 'integration-test', 'end-to-end-test']); + +const ESCALATIONS = new Set(['NONE', 'BROADEN', 'FULL', 'INVALIDATE']); + +export class InputError extends Error { + constructor(issues, code = 'INVALID_INPUT') { + const sorted = [...new Set(issues)].sort(); + super(sorted.join('; ')); + this.name = 'InputError'; + this.code = code; + this.issues = sorted; + } +} + +function objectAt(value, path, issues) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + issues.push(`${path} must be an object`); + return {}; + } + return value; +} + +function arrayAt(value, path, issues, { nonempty = false } = {}) { + if (!Array.isArray(value)) { + issues.push(`${path} must be an array`); + return []; + } + if (nonempty && value.length === 0) { + issues.push(`${path} must not be empty`); + } + return value; +} + +function stringAt(value, path, issues) { + if (typeof value !== 'string' || value.trim() === '') { + issues.push(`${path} must be a nonempty string`); + return ''; + } + return value; +} + +function stringArray(value, path, issues) { + const result = arrayAt(value, path, issues); + result.forEach((item, index) => stringAt(item, `${path}[${index}]`, issues)); + return result; +} + +function rejectUnknownKeys(object, allowed, path, issues) { + for (const key of Object.keys(object)) { + if (!allowed.includes(key)) { + issues.push(`${path}.${key} is not allowed`); + } + } +} + +function uniqueBy(items, key, path, issues) { + const seen = new Set(); + for (const [index, item] of items.entries()) { + const value = item?.[key]; + if (typeof value === 'string' && seen.has(value)) { + issues.push(`${path}[${index}].${key} duplicates ${value}`); + } + seen.add(value); + } +} + +function validateChange(raw, issues) { + const change = objectAt(raw, 'change', issues); + rejectUnknownKeys(change, ['base_revision', 'target_revision', 'identity', 'paths'], 'change', issues); + stringAt(change.base_revision, 'change.base_revision', issues); + stringAt(change.target_revision, 'change.target_revision', issues); + if (change.identity !== undefined) stringAt(change.identity, 'change.identity', issues); + const paths = arrayAt(change.paths, 'change.paths', issues, { nonempty: true }); + paths.forEach((rawPath, index) => { + const path = objectAt(rawPath, `change.paths[${index}]`, issues); + rejectUnknownKeys(path, ['path', 'regions', 'risk_tags'], `change.paths[${index}]`, issues); + stringAt(path.path, `change.paths[${index}].path`, issues); + stringArray(path.regions ?? [], `change.paths[${index}].regions`, issues); + stringArray(path.risk_tags ?? [], `change.paths[${index}].risk_tags`, issues); + }); + uniqueBy(paths, 'path', 'change.paths', issues); + return change; +} + +function validateEvidence(raw, changedPaths, issues) { + const evidence = objectAt(raw, 'evidence', issues); + rejectUnknownKeys(evidence, ['identity', 'complete', 'providers', 'components', 'impacts'], 'evidence', issues); + stringAt(evidence.identity, 'evidence.identity', issues); + if (typeof evidence.complete !== 'boolean') issues.push('evidence.complete must be a boolean'); + const providers = arrayAt(evidence.providers, 'evidence.providers', issues, { nonempty: true }); + providers.forEach((rawProvider, index) => { + const provider = objectAt(rawProvider, `evidence.providers[${index}]`, issues); + rejectUnknownKeys(provider, ['id', 'kind', 'version', 'identity'], `evidence.providers[${index}]`, issues); + for (const key of ['id', 'kind', 'version', 'identity']) { + stringAt(provider[key], `evidence.providers[${index}].${key}`, issues); + } + }); + uniqueBy(providers, 'id', 'evidence.providers', issues); + + const components = arrayAt(evidence.components, 'evidence.components', issues, { nonempty: true }); + components.forEach((rawComponent, index) => { + const component = objectAt(rawComponent, `evidence.components[${index}]`, issues); + rejectUnknownKeys(component, ['id', 'dependencies'], `evidence.components[${index}]`, issues); + stringAt(component.id, `evidence.components[${index}].id`, issues); + stringArray(component.dependencies, `evidence.components[${index}].dependencies`, issues); + }); + uniqueBy(components, 'id', 'evidence.components', issues); + const componentIds = new Set(components.map((item) => item.id)); + components.forEach((component) => component.dependencies?.forEach((dependency) => { + if (!componentIds.has(dependency)) { + issues.push(`component ${component.id} references unknown dependency ${dependency}`); + } + })); + + const impacts = arrayAt(evidence.impacts, 'evidence.impacts', issues); + impacts.forEach((rawImpact, index) => { + const impact = objectAt(rawImpact, `evidence.impacts[${index}]`, issues); + rejectUnknownKeys(impact, ['path', 'components', 'confidence', 'reason'], `evidence.impacts[${index}]`, issues); + stringAt(impact.path, `evidence.impacts[${index}].path`, issues); + stringArray(impact.components, `evidence.impacts[${index}].components`, issues); + if (!['KNOWN', 'UNKNOWN'].includes(impact.confidence)) { + issues.push(`evidence.impacts[${index}].confidence must be KNOWN or UNKNOWN`); + } + if (impact.reason !== undefined) stringAt(impact.reason, `evidence.impacts[${index}].reason`, issues); + if (impact.confidence === 'KNOWN' && impact.components?.length === 0) { + issues.push(`evidence.impacts[${index}].components must not be empty when confidence is KNOWN`); + } + impact.components?.forEach((component) => { + if (!componentIds.has(component)) { + issues.push(`impact for ${impact.path} references unknown component ${component}`); + } + }); + if (!changedPaths.has(impact.path)) { + issues.push(`impact path ${impact.path} is not in the change set`); + } + }); + uniqueBy(impacts, 'path', 'evidence.impacts', issues); + return evidence; +} + +function validateCatalog(raw, componentIds, issues) { + const catalog = objectAt(raw, 'catalog', issues); + rejectUnknownKeys(catalog, ['identity', 'complete', 'checks'], 'catalog', issues); + stringAt(catalog.identity, 'catalog.identity', issues); + if (typeof catalog.complete !== 'boolean') issues.push('catalog.complete must be a boolean'); + const checks = arrayAt(catalog.checks, 'catalog.checks', issues, { nonempty: true }); + checks.forEach((rawCheck, index) => { + const check = objectAt(rawCheck, `catalog.checks[${index}]`, issues); + rejectUnknownKeys(check, ['id', 'type', 'command', 'scope', 'tags', 'test_executions', 'cost'], `catalog.checks[${index}]`, issues); + stringAt(check.id, `catalog.checks[${index}].id`, issues); + if (!CHECK_TYPES.has(check.type)) issues.push(`catalog.checks[${index}].type is unsupported`); + stringAt(check.command, `catalog.checks[${index}].command`, issues); + stringArray(check.tags, `catalog.checks[${index}].tags`, issues); + if (!Number.isSafeInteger(check.test_executions) || check.test_executions < 0) { + issues.push(`catalog.checks[${index}].test_executions must be a nonnegative safe integer`); + } + if (!TEST_CHECK_TYPES.has(check.type) && check.test_executions !== 0) { + issues.push(`catalog.checks[${index}] non-test check cannot declare test executions`); + } + const scope = objectAt(check.scope, `catalog.checks[${index}].scope`, issues); + rejectUnknownKeys(scope, ['components'], `catalog.checks[${index}].scope`, issues); + const scoped = stringArray(scope.components, `catalog.checks[${index}].scope.components`, issues); + if (scoped.length === 0) issues.push(`catalog.checks[${index}].scope.components must not be empty`); + scoped.forEach((component) => { + if (component !== '*' && !componentIds.has(component)) { + issues.push(`check ${check.id} references unknown component ${component}`); + } + }); + if (check.cost !== undefined) { + const cost = objectAt(check.cost, `catalog.checks[${index}].cost`, issues); + rejectUnknownKeys(cost, ['value', 'unit'], `catalog.checks[${index}].cost`, issues); + if (!Number.isFinite(cost.value) || cost.value < 0) issues.push(`catalog.checks[${index}].cost.value must be finite and nonnegative`); + stringAt(cost.unit, `catalog.checks[${index}].cost.unit`, issues); + } + }); + uniqueBy(checks, 'id', 'catalog.checks', issues); + return catalog; +} + +function validatePolicy(raw, issues) { + const policy = objectAt(raw, 'policy', issues); + rejectUnknownKeys(policy, ['identity', 'version', 'rules'], 'policy', issues); + stringAt(policy.identity, 'policy.identity', issues); + stringAt(policy.version, 'policy.version', issues); + const rules = arrayAt(policy.rules, 'policy.rules', issues); + rules.forEach((rawRule, index) => { + const rule = objectAt(rawRule, `policy.rules[${index}]`, issues); + rejectUnknownKeys(rule, ['id', 'match', 'escalation', 'required_check_tags'], `policy.rules[${index}]`, issues); + stringAt(rule.id, `policy.rules[${index}].id`, issues); + if (!ESCALATIONS.has(rule.escalation)) issues.push(`policy.rules[${index}].escalation is invalid`); + stringArray(rule.required_check_tags, `policy.rules[${index}].required_check_tags`, issues); + const match = objectAt(rule.match, `policy.rules[${index}].match`, issues); + rejectUnknownKeys(match, ['path_globs', 'risk_tags', 'component_ids'], `policy.rules[${index}].match`, issues); + const matcherCount = ['path_globs', 'risk_tags', 'component_ids'] + .map((key) => stringArray(match[key] ?? [], `policy.rules[${index}].match.${key}`, issues).length) + .reduce((sum, count) => sum + count, 0); + if (matcherCount === 0) issues.push(`policy.rules[${index}].match must contain at least one matcher`); + }); + uniqueBy(rules, 'id', 'policy.rules', issues); + return policy; +} + +export function validateInput(input) { + const issues = []; + const root = objectAt(input, 'input', issues); + rejectUnknownKeys(root, ['schema', 'change', 'evidence', 'catalog', 'policy'], 'input', issues); + if (root.schema !== INPUT_SCHEMA) issues.push(`input.schema must be ${INPUT_SCHEMA}`); + const change = validateChange(root.change, issues); + const changedPaths = new Set((change.paths ?? []).map((item) => item.path)); + const evidence = validateEvidence(root.evidence, changedPaths, issues); + const componentIds = new Set((evidence.components ?? []).map((item) => item.id)); + validateCatalog(root.catalog, componentIds, issues); + validatePolicy(root.policy, issues); + if (issues.length) throw new InputError(issues); + return root; +} + +export function validateShadowInput(input, plan) { + const issues = []; + const root = objectAt(input, 'shadow', issues); + rejectUnknownKeys(root, ['schema', 'change_identity', 'executed_check_ids', 'failures'], 'shadow', issues); + if (root.schema !== SHADOW_INPUT_SCHEMA) issues.push(`shadow.schema must be ${SHADOW_INPUT_SCHEMA}`); + if (root.change_identity !== plan.change.identity) issues.push('shadow.change_identity does not match plan'); + const known = new Set([...plan.selected_checks, ...plan.skipped_checks].map((item) => item.id)); + const executed = stringArray(root.executed_check_ids, 'shadow.executed_check_ids', issues); + if (new Set(executed).size !== executed.length) issues.push('shadow.executed_check_ids contains duplicates'); + executed.forEach((id) => { + if (!known.has(id)) issues.push(`shadow executed unknown check ${id}`); + }); + const failures = arrayAt(root.failures, 'shadow.failures', issues); + failures.forEach((rawFailure, index) => { + const failure = objectAt(rawFailure, `shadow.failures[${index}]`, issues); + rejectUnknownKeys(failure, ['check_id', 'relevant', 'reason'], `shadow.failures[${index}]`, issues); + stringAt(failure.check_id, `shadow.failures[${index}].check_id`, issues); + if (typeof failure.relevant !== 'boolean') issues.push(`shadow.failures[${index}].relevant must be a boolean`); + stringAt(failure.reason, `shadow.failures[${index}].reason`, issues); + if (!executed.includes(failure.check_id)) issues.push(`failure ${failure.check_id} was not executed`); + }); + uniqueBy(failures, 'check_id', 'shadow.failures', issues); + if (issues.length) throw new InputError(issues); + return root; +} diff --git a/src/value-receipt.js b/src/value-receipt.js new file mode 100644 index 0000000..d53c04f --- /dev/null +++ b/src/value-receipt.js @@ -0,0 +1,96 @@ +const RECEIPT_SCHEMA = 'opsle.value-receipt.v1'; + +function measurement({ id, baseline, result, delta, unit, direction, operatorDisplay, limitation = [] }) { + return { + aggregation: { safe: unit === 'count', method: unit === 'count' ? 'SUM' : null }, + baseline, + class: 'EXACT', + delta, + derivation: null, + direction, + evidence_refs: ['verification_plan'], + id, + limitations: limitation, + operator_display: operatorDisplay, + result, + source_verification: 'VERIFIED', + unit, + }; +} + +function isTest(check) { + return ['unit-test', 'integration-test', 'end-to-end-test'].includes(check.type); +} + +export function buildValueReceipt(plan, { mechanismRevision = null, runId = null } = {}) { + const all = [...plan.selected_checks, ...plan.skipped_checks]; + const availableTests = all.filter(isTest).reduce((sum, item) => sum + item.test_executions, 0); + const selectedTests = plan.selected_checks.filter(isTest).reduce((sum, item) => sum + item.test_executions, 0); + const skippedTests = availableTests - selectedTests; + const ratio = availableTests === 0 ? null : `${skippedTests}/${availableTests}`; + return { + schema: RECEIPT_SCHEMA, + mechanism: { + id: 'opsle.affected-verification', + name: 'Affected Verification', + version: '0.1.0', + revision: mechanismRevision, + }, + run: { id: runId, repository: null, task_classification: null, work_classification: 'DETERMINISTIC_VERIFICATION_PLANNING' }, + operation: { + id: plan.plan_identity, + name: 'verification-planning', + configuration_id: plan.provenance.verification_catalog_identity, + policy_id: plan.provenance.policy_identity, + }, + measurements: [ + measurement({ id: 'checks_available', baseline: null, result: all.length, delta: null, unit: 'count', direction: 'NEUTRAL', operatorDisplay: false }), + measurement({ id: 'checks_selected', baseline: null, result: plan.selected_checks.length, delta: null, unit: 'count', direction: 'LOWER_IS_VALUE', operatorDisplay: true }), + measurement({ id: 'checks_skipped', baseline: null, result: plan.skipped_checks.length, delta: null, unit: 'count', direction: 'HIGHER_IS_VALUE', operatorDisplay: true }), + measurement({ id: 'test_executions_available', baseline: null, result: availableTests, delta: null, unit: 'count', direction: 'NEUTRAL', operatorDisplay: false }), + measurement({ id: 'test_executions_selected', baseline: null, result: selectedTests, delta: null, unit: 'count', direction: 'LOWER_IS_VALUE', operatorDisplay: true }), + measurement({ id: 'test_executions_skipped', baseline: null, result: skippedTests, delta: null, unit: 'count', direction: 'HIGHER_IS_VALUE', operatorDisplay: true }), + measurement({ + id: 'test_execution_reduction', + baseline: null, + result: ratio, + delta: null, + unit: 'ratio', + direction: 'HIGHER_IS_VALUE', + operatorDisplay: true, + limitation: availableTests === 0 + ? ['Unavailable because no test executions are declared.'] + : ['Computed only across catalog entries measured in test executions; unlike verification types are not combined.'], + }), + ], + evidence: [{ id: 'verification_plan', kind: 'CONTENT_HASH', locator: plan.plan_identity, trust: 'VERIFIED' }], + limitations: [ + 'No time, token, cost, correctness, or causal savings claim is made.', + 'Counts describe the configured catalog and declared test executions, not observed execution.', + 'A skipped check count is value only within the plan sufficiency and uncertainty classification.', + ], + extensions: { + sufficiency: plan.sufficiency, + impact_uncertainty: plan.uncertainty.state, + selected_by_type: Object.fromEntries( + [...new Set(plan.selected_checks.map((item) => item.type))].sort().map((type) => [ + type, + plan.selected_checks.filter((item) => item.type === type).length, + ]), + ), + }, + }; +} + +export function operatorIndicator(plan) { + const all = [...plan.selected_checks, ...plan.skipped_checks]; + const availableTests = all.filter(isTest).reduce((sum, item) => sum + item.test_executions, 0); + const selectedTests = plan.selected_checks.filter(isTest).reduce((sum, item) => sum + item.test_executions, 0); + const skippedTests = availableTests - selectedTests; + const other = Object.entries(plan.selected_checks.filter((item) => !isTest(item)).reduce((counts, item) => { + counts[item.type] = (counts[item.type] ?? 0) + 1; + return counts; + }, {})).sort().map(([type, count]) => `${count} ${type}`).join(', ') || 'none'; + const reduction = availableTests === 0 ? 'n/a' : `${((skippedTests / availableTests) * 100).toFixed(1)}%`; + return `[Affected Verification] Selected tests: ${selectedTests}/${availableTests}; other checks: ${other}; skipped tests: ${skippedTests}; test-execution reduction: ${reduction}; sufficiency: ${plan.sufficiency}; impact uncertainty: ${plan.uncertainty.state.toLowerCase()}`; +} diff --git a/tests/cli.test.js b/tests/cli.test.js new file mode 100644 index 0000000..95997d6 --- /dev/null +++ b/tests/cli.test.js @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { fixture } from '../fixtures/scenarios.js'; + +const bin = fileURLToPath(new URL('../bin/affected-verification.js', import.meta.url)); + +test('CLI keeps canonical plan on stdout, indicator on stderr, and receipt in sidecar', () => { + const dir = mkdtempSync(join(tmpdir(), 'affected-verification-')); + const inputPath = join(dir, 'input.json'); + const receiptPath = join(dir, 'receipt.json'); + writeFileSync(inputPath, JSON.stringify(fixture('isolated-implementation'))); + const result = spawnSync(process.execPath, [bin, 'plan', inputPath, '--receipt', receiptPath], { encoding: 'utf8' }); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).schema, 'opsle.affected-verification.plan.v1'); + assert.match(result.stderr, /^\[Affected Verification\]/); + assert.equal(JSON.parse(readFileSync(receiptPath, 'utf8')).schema, 'opsle.value-receipt.v1'); +}); + +test('fixture CLI is deterministic across runs', () => { + const first = spawnSync(process.execPath, [bin, 'fixture', 'readme-only'], { encoding: 'utf8' }); + const second = spawnSync(process.execPath, [bin, 'fixture', 'readme-only'], { encoding: 'utf8' }); + assert.equal(first.status, 0); + assert.equal(first.stdout, second.stdout); + assert.equal(first.stderr, second.stderr); +}); + +test('malformed JSON returns a machine-readable error and no success indicator', () => { + const dir = mkdtempSync(join(tmpdir(), 'affected-verification-')); + const inputPath = join(dir, 'bad.json'); + writeFileSync(inputPath, '{bad'); + const result = spawnSync(process.execPath, [bin, 'plan', inputPath], { encoding: 'utf8' }); + assert.equal(result.status, 2); + assert.equal(JSON.parse(result.stdout).code, 'INVALID_JSON'); + assert.equal(result.stderr, ''); +}); + +test('critical metadata returns PLAN_INVALIDATED and no success indicator', () => { + const result = spawnSync(process.execPath, [bin, 'fixture', 'critical-verification-metadata'], { encoding: 'utf8' }); + assert.equal(result.status, 2); + assert.equal(JSON.parse(result.stdout).code, 'PLAN_INVALIDATED'); + assert.equal(result.stderr, ''); +}); diff --git a/tests/planner.test.js b/tests/planner.test.js new file mode 100644 index 0000000..904d78e --- /dev/null +++ b/tests/planner.test.js @@ -0,0 +1,204 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { contentIdentity, InputError, planVerification } from '../src/index.js'; +import { fixture, scenarios } from '../fixtures/scenarios.js'; +import { negativeCases } from '../fixtures/negative-cases.js'; + +for (const scenario of scenarios) { + test(`fixture: ${scenario.id}`, () => { + if (scenario.expected_error) { + assert.throws( + () => planVerification(scenario.input), + (error) => error instanceof InputError && error.code === scenario.expected_error, + ); + return; + } + const plan = planVerification(scenario.input); + assert.equal(plan.sufficiency, scenario.expected_sufficiency); + assert.equal(plan.argument.unknown_is_safe_to_skip, false); + assert.equal(plan.argument.every_skip_explained, true); + }); +} + +for (const negative of negativeCases) { + test(`negative fixture: ${negative.id}`, () => { + assert.throws( + () => planVerification(negative.input), + (error) => error instanceof InputError && error.code === negative.expected_error, + ); + }); +} + +test('identical input produces an identical deterministic plan', () => { + const input = fixture('isolated-implementation'); + assert.deepEqual(planVerification(input), planVerification(structuredClone(input))); +}); + +test('unrelated checks are skipped with exact reasons', () => { + const plan = planVerification(fixture('isolated-implementation')); + const skipped = plan.skipped_checks.find((item) => item.id === 'unrelated.large-suite'); + assert.deepEqual(skipped.reasons.map((item) => item.code), ['OUTSIDE_TRANSITIVE_IMPACT_SET']); +}); + +test('direct affected checks are selected', () => { + const plan = planVerification(fixture('isolated-implementation')); + assert.deepEqual( + plan.selected_checks.map((item) => item.id), + ['isolated.unit', 'project.lint', 'project.typecheck'], + ); +}); + +test('reverse dependencies broaden the selected components and checks', () => { + const plan = planVerification(fixture('reverse-dependent')); + assert.deepEqual(plan.affected_components, [ + { id: 'consumer', direct: false, via_dependencies: ['feature'] }, + { id: 'feature', direct: true, via_dependencies: [] }, + ]); + assert.ok(plan.selected_checks.some((item) => item.id === 'consumer.integration')); +}); + +test('shared component policy broadens deterministically', () => { + const plan = planVerification(fixture('shared-common')); + assert.equal(plan.escalation.state, 'BROADENED'); + assert.deepEqual(plan.risk.matched_policy_rules, ['shared-boundary']); + assert.ok(plan.selected_checks.some((item) => item.id === 'subsystem-a.integration')); + assert.ok(plan.selected_checks.some((item) => item.id === 'subsystem-b.integration')); +}); + +test('global policy selects the full catalog', () => { + const plan = planVerification(fixture('global-configuration')); + assert.equal(plan.skipped_checks.length, 0); + assert.equal(plan.escalation.state, 'FULL'); + assert.equal(plan.selected_checks.length, fixture('global-configuration').catalog.checks.length); +}); + +test('incomplete impact evidence never claims targeted sufficiency', () => { + const plan = planVerification(fixture('incomplete-dependency-graph')); + assert.equal(plan.sufficiency, 'FULL_VERIFICATION_REQUIRED'); + assert.equal(plan.uncertainty.state, 'PRESENT'); + assert.ok(plan.uncertainty.reasons.includes('DEPENDENCY_EVIDENCE_INCOMPLETE')); + assert.equal(plan.skipped_checks.length, 0); +}); + +test('unknown changed path fails closed', () => { + const plan = planVerification(fixture('unknown-changed-file')); + assert.equal(plan.sufficiency, 'FULL_VERIFICATION_REQUIRED'); + assert.deepEqual(plan.uncertainty.reasons, ['UNKNOWN_IMPACT:mystery/generated.xyz']); + assert.equal(plan.skipped_checks.length, 0); +}); + +test('incomplete catalog is insufficient even when all known checks are selected', () => { + const input = fixture('isolated-implementation'); + input.catalog.complete = false; + const plan = planVerification(input); + assert.equal(plan.sufficiency, 'INSUFFICIENT_EVIDENCE'); + assert.equal(plan.skipped_checks.length, 0); + assert.ok(plan.uncertainty.reasons.includes('VERIFICATION_CATALOG_INCOMPLETE')); +}); + +test('uncovered affected component is insufficient even with a complete catalog', () => { + const input = fixture('isolated-implementation'); + input.catalog.checks = input.catalog.checks.filter((check) => !check.scope.components.includes('isolated')); + const plan = planVerification(input); + assert.equal(plan.sufficiency, 'INSUFFICIENT_EVIDENCE'); + assert.ok(plan.uncertainty.reasons.includes('NO_VERIFICATION_COVERAGE:isolated')); + assert.equal(plan.skipped_checks.length, 0); +}); + +test('known impact cannot assert an empty component set', () => { + const input = fixture('isolated-implementation'); + input.evidence.impacts[0].components = []; + assert.throws(() => planVerification(input), /must not be empty when confidence is KNOWN/); +}); + +test('unsatisfied policy requirement is insufficient and fails closed', () => { + const input = fixture('authentication-boundary'); + input.catalog.checks.forEach((check) => { + check.tags = check.tags.filter((tag) => tag !== 'security-boundary'); + }); + const plan = planVerification(input); + assert.equal(plan.sufficiency, 'INSUFFICIENT_EVIDENCE'); + assert.ok(plan.uncertainty.reasons.includes('POLICY_REQUIREMENT_UNSATISFIED:security-boundary:security-boundary')); + assert.equal(plan.skipped_checks.length, 0); +}); + +test('every skipped check has a nonempty evidence-backed reason', () => { + const plan = planVerification(fixture('readme-only')); + for (const check of plan.skipped_checks) { + assert.ok(check.reasons.length > 0); + assert.ok(check.reasons.every((item) => item.code && item.detail && item.evidence_refs.length)); + } +}); + +test('duplicate impact evidence is rejected instead of merged optimistically', () => { + const input = fixture('isolated-implementation'); + input.evidence.impacts.push({ ...input.evidence.impacts[0], components: ['unrelated'] }); + assert.throws( + () => planVerification(input), + (error) => error instanceof InputError && error.issues.some((item) => item.includes('duplicates')), + ); +}); + +test('duplicate provider evidence is rejected', () => { + const input = fixture('isolated-implementation'); + input.evidence.providers.push({ ...input.evidence.providers[0] }); + assert.throws(() => planVerification(input), /duplicates synthetic-graph/); +}); + +test('unknown component dependency is rejected', () => { + const input = fixture('isolated-implementation'); + input.evidence.components[0].dependencies.push('missing'); + assert.throws(() => planVerification(input), /unknown dependency missing/); +}); + +test('malformed schema and unknown fields are rejected', () => { + const input = fixture('isolated-implementation'); + input.schema = 'wrong'; + input.unsafe = true; + assert.throws( + () => planVerification(input), + (error) => error.issues.length === 2 && error.issues[0].includes('must be opsle') && error.issues[1].includes('not allowed'), + ); +}); + +test('negative numeric inputs are rejected', () => { + const input = fixture('isolated-implementation'); + input.catalog.checks[0].test_executions = -1; + assert.throws(() => planVerification(input), /nonnegative safe integer/); +}); + +test('unsafe integer inputs are rejected', () => { + const input = fixture('isolated-implementation'); + input.catalog.checks[1].test_executions = Number.MAX_SAFE_INTEGER + 1; + assert.throws(() => planVerification(input), /nonnegative safe integer/); +}); + +test('non-test checks cannot disguise test execution counts', () => { + const input = fixture('isolated-implementation'); + input.catalog.checks[0].test_executions = 1; + assert.throws(() => planVerification(input), /non-test check cannot declare test executions/); +}); + +test('test-infrastructure checks cannot declare test executions', () => { + const input = fixture('isolated-implementation'); + const check = input.catalog.checks.find((item) => item.type === 'test-infrastructure'); + check.test_executions = 1; + assert.throws(() => planVerification(input), /non-test check cannot declare test executions/); +}); + +test('plan identity and provenance hash are stable and content-bound', () => { + const input = fixture('isolated-implementation'); + const plan = planVerification(input); + const withoutIdentity = { ...plan, plan_identity: undefined }; + assert.equal(plan.plan_identity, contentIdentity(withoutIdentity)); + assert.equal(plan.provenance.input_hash, contentIdentity(input)); + const changed = fixture('isolated-implementation'); + changed.change.target_revision = 'different-target'; + assert.notEqual(planVerification(changed).plan_identity, plan.plan_identity); +}); + +test('selected and skipped arrays are immutable', () => { + const plan = planVerification(fixture('isolated-implementation')); + assert.throws(() => plan.selected_checks.push({}), TypeError); + assert.throws(() => plan.skipped_checks[0].reasons.push({}), TypeError); +}); diff --git a/tests/value-shadow.test.js b/tests/value-shadow.test.js new file mode 100644 index 0000000..2432de2 --- /dev/null +++ b/tests/value-shadow.test.js @@ -0,0 +1,120 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + buildValueReceipt, + classifyShadow, + operatorIndicator, + planVerification, +} from '../src/index.js'; +import { fixture } from '../fixtures/scenarios.js'; +import { tamperPlan } from '../fixtures/negative-cases.js'; + +function fullRun(plan, failures = []) { + return { + schema: 'opsle.affected-verification.shadow-input.v1', + change_identity: plan.change.identity, + executed_check_ids: [...plan.selected_checks, ...plan.skipped_checks].map((item) => item.id), + failures, + }; +} + +test('Visible Value calculations are exact for the large unrelated suite', () => { + const plan = planVerification(fixture('unrelated-large-suite')); + const receipt = buildValueReceipt(plan); + const values = Object.fromEntries(receipt.measurements.map((item) => [item.id, item.result])); + assert.equal(values.checks_available, 4); + assert.equal(values.checks_selected, 3); + assert.equal(values.checks_skipped, 1); + assert.equal(values.test_executions_available, 1043); + assert.equal(values.test_executions_selected, 14); + assert.equal(values.test_executions_skipped, 1029); + assert.equal(values.test_execution_reduction, '1029/1043'); + assert.equal(receipt.schema, 'opsle.value-receipt.v1'); + assert.equal(receipt.evidence[0].locator, plan.plan_identity); +}); + +test('operator indicator never aggregates unlike checks into its test percentage', () => { + const plan = planVerification(fixture('unrelated-large-suite')); + assert.equal( + operatorIndicator(plan), + '[Affected Verification] Selected tests: 14/1043; other checks: 1 lint, 1 typecheck; skipped tests: 1029; test-execution reduction: 98.7%; sufficiency: SUFFICIENT_TARGETED; impact uncertainty: none', + ); +}); + +test('receipt exposes no unmeasured time token cost or correctness savings', () => { + const receipt = buildValueReceipt(planVerification(fixture('isolated-implementation'))); + assert.ok(receipt.limitations[0].includes('No time, token, cost, correctness')); + assert.ok(receipt.measurements.every((item) => !['token', 'usd', 'millisecond'].includes(item.unit))); +}); + +test('shadow mode classifies a relevant skipped failure as a selection miss', () => { + const plan = planVerification(fixture('isolated-implementation')); + const observation = classifyShadow(plan, fullRun(plan, [{ + check_id: 'unrelated.large-suite', + relevant: true, + reason: 'Full execution exposed a regression causally linked to the change.', + }])); + assert.equal(observation.classification, 'SELECTION_MISS'); + assert.deepEqual(observation.selection_misses, [{ + check_id: 'unrelated.large-suite', + exact_miss_reason: 'Full execution exposed a regression causally linked to the change.', + }]); +}); + +test('shadow mode classifies a complete clean run as no miss', () => { + const plan = planVerification(fixture('isolated-implementation')); + assert.equal(classifyShadow(plan, fullRun(plan)).classification, 'NO_SELECTION_MISS'); +}); + +test('failure in a selected check is not a selection miss', () => { + const plan = planVerification(fixture('isolated-implementation')); + const observation = classifyShadow(plan, fullRun(plan, [{ + check_id: 'isolated.unit', relevant: true, reason: 'Selected check failed.', + }])); + assert.equal(observation.classification, 'NO_SELECTION_MISS'); + assert.deepEqual(observation.selection_misses, []); +}); + +test('irrelevant skipped failure is recorded but not misclassified as a miss', () => { + const plan = planVerification(fixture('isolated-implementation')); + const observation = classifyShadow(plan, fullRun(plan, [{ + check_id: 'unrelated.large-suite', relevant: false, reason: 'Known unrelated flaky failure.', + }])); + assert.equal(observation.classification, 'NO_SELECTION_MISS'); + assert.equal(observation.full_run_failures.length, 1); +}); + +test('incomplete full execution is indeterminate', () => { + const plan = planVerification(fixture('isolated-implementation')); + const shadow = fullRun(plan); + shadow.executed_check_ids.pop(); + assert.equal(classifyShadow(plan, shadow).classification, 'INDETERMINATE_FULL_RUN_INCOMPLETE'); +}); + +test('shadow observation identity is stable', () => { + const plan = planVerification(fixture('isolated-implementation')); + assert.deepEqual(classifyShadow(plan, fullRun(plan)), classifyShadow(plan, fullRun(plan))); +}); + +test('shadow input rejects unknown checks and mismatched changes', () => { + const plan = planVerification(fixture('isolated-implementation')); + const shadow = fullRun(plan); + shadow.change_identity = 'wrong'; + shadow.executed_check_ids.push('unknown'); + assert.throws(() => classifyShadow(plan, shadow), /shadow executed unknown check unknown; shadow.change_identity does not match plan/); +}); + +test('shadow input rejects a tampered plan identity', () => { + const plan = planVerification(fixture('isolated-implementation')); + assert.throws( + () => classifyShadow(tamperPlan(plan), fullRun(plan)), + (error) => error.code === 'TAMPERED_PLAN' && error.issues.includes('plan identity does not match canonical content'), + ); +}); + +test('shadow input rejects malformed plan structure', () => { + assert.throws( + () => classifyShadow(null, {}), + (error) => error.code === 'TAMPERED_PLAN' && error.issues.includes('plan must be an object'), + ); +}); diff --git a/tools/conformance.js b/tools/conformance.js new file mode 100755 index 0000000..c733dd0 --- /dev/null +++ b/tools/conformance.js @@ -0,0 +1,29 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { InputError, buildValueReceipt, planVerification } from '../src/index.js'; +import { scenarios } from '../fixtures/scenarios.js'; +import { negativeCases } from '../fixtures/negative-cases.js'; + +let passed = 0; +for (const scenario of scenarios) { + if (scenario.expected_error) { + assert.throws( + () => planVerification(scenario.input), + (error) => error instanceof InputError && error.code === scenario.expected_error, + ); + } else { + const plan = planVerification(scenario.input); + assert.equal(plan.sufficiency, scenario.expected_sufficiency); + assert.ok(plan.skipped_checks.every((check) => check.reasons.length > 0)); + assert.equal(buildValueReceipt(plan).schema, 'opsle.value-receipt.v1'); + } + passed += 1; +} +for (const negative of negativeCases) { + assert.throws( + () => planVerification(negative.input), + (error) => error instanceof InputError && error.code === negative.expected_error, + ); + passed += 1; +} +process.stdout.write(`PASS: ${passed} synthetic conformance scenarios\n`); From fda1bc9ec1408aa7543f52a03aa1b06841ac5e96 Mon Sep 17 00:00:00 2001 From: Deploy Date: Mon, 31 Aug 2026 01:26:34 +0000 Subject: [PATCH 2/3] chore: normalize public source whitespace --- .github/workflows/ci.yml | 1 - .gitignore | 1 - BENCHMARK.md | 1 - CONTRIBUTING.md | 1 - LIMITATIONS.md | 1 - PRIOR_ART.md | 1 - SECURITY.md | 1 - SPEC.md | 4 ++-- fixtures/negative-cases.js | 1 - package.json | 1 - src/canonical.js | 1 - src/index.js | 1 - 12 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78574c2..6fca239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,4 +28,3 @@ jobs: with: fetch-depth: 0 - uses: gitleaks/gitleaks-action@v2 - diff --git a/.gitignore b/.gitignore index 35e3dfa..bbe895a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ node_modules/ coverage/ *.tmp - diff --git a/BENCHMARK.md b/BENCHMARK.md index e1d33bf..1803686 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -62,4 +62,3 @@ Unlike verification types are not added into one deceptive percentage. Counterfa ## Initial experiment sequence First, freeze a real public repository with a trustworthy full-verification baseline and run in `OBSERVE`/`SHADOW`. Compare plan identities and full outcomes without replacing CI. Only after the oracle, catalog completeness, adapter fidelity, and miss classifications are independently reviewable should a bounded trust decision be considered. - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f79956b..1744b4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,3 @@ # Contributing Keep the core deterministic, dependency-light, and separate from execution. Changes to selection semantics require exact tests, updated contract text, and new or revised conformance fixtures. Prior-art claims require primary sources. Do not add benchmark results without immutable inputs, a full-verification oracle, and explicit failure reporting. - diff --git a/LIMITATIONS.md b/LIMITATIONS.md index c4cf3c5..07e75b8 100644 --- a/LIMITATIONS.md +++ b/LIMITATIONS.md @@ -12,4 +12,3 @@ - No production trust stage is justified. This repository is at most `PROTOTYPED`. - No time, token, monetary, correctness, failure-prevention, or causal savings claim is supported. - Context Firewall, Decision Evidence, Agent Trajectory Profiler, Gearbox, and Opsle Tasks are external consumers or validators, not dependencies. - diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 6410697..66391d5 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -25,4 +25,3 @@ The prototype's defensible distinction is narrower: it combines potentially hete ## Novelty ceiling No novelty is claimed. Future comparison must include full verification, native affected tooling, Affected Verification, and Affected Verification consuming native tooling. Value exists only if the composed argument improves workload without increasing relevant misses under controlled shadow evidence. - diff --git a/SECURITY.md b/SECURITY.md index 2759593..3dc3431 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -3,4 +3,3 @@ Please report vulnerabilities through GitHub's private vulnerability reporting for `opsle/affected-verification` when available, or contact the repository maintainers privately. The prototype does not execute catalog commands, access a network, load plugins, or evaluate model output. Treat normalized evidence, catalogs, policies, and shadow relevance as untrusted input. A plan must not authorize execution by itself. - diff --git a/SPEC.md b/SPEC.md index cb822cc..04b5800 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,7 +1,7 @@ # Affected Verification specification -Status: normative prototype contract -Plan identity: `opsle.affected-verification.plan.v1` +Status: normative prototype contract +Plan identity: `opsle.affected-verification.plan.v1` Input identity: `opsle.affected-verification.input.v1` ## 1. Canonical definition diff --git a/fixtures/negative-cases.js b/fixtures/negative-cases.js index a710a4c..1504eaf 100644 --- a/fixtures/negative-cases.js +++ b/fixtures/negative-cases.js @@ -29,4 +29,3 @@ export function tamperPlan(plan) { tampered.selected_checks[0].command = 'changed-after-planning'; return tampered; } - diff --git a/package.json b/package.json index c1ced78..2c89c99 100644 --- a/package.json +++ b/package.json @@ -19,4 +19,3 @@ }, "license": "Apache-2.0" } - diff --git a/src/canonical.js b/src/canonical.js index 6ca7476..cbccf15 100644 --- a/src/canonical.js +++ b/src/canonical.js @@ -32,4 +32,3 @@ export function deepFreeze(value) { } return value; } - diff --git a/src/index.js b/src/index.js index 31725b1..9b1a431 100644 --- a/src/index.js +++ b/src/index.js @@ -9,4 +9,3 @@ export { SHADOW_OBSERVATION_SCHEMA, InputError, } from './validate.js'; - From 8fbe28f4958303256cf41d3b99aa456d1dc92c71 Mon Sep 17 00:00:00 2001 From: Deploy Date: Mon, 31 Aug 2026 01:27:41 +0000 Subject: [PATCH 3/3] ci: run pinned license-free secret scan --- .github/workflows/ci.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fca239..bc91c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,4 +27,16 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: gitleaks/gitleaks-action@v2 + - name: Install pinned gitleaks + env: + GITLEAKS_VERSION: 8.30.1 + run: | + base="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + archive="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + checksums="gitleaks_${GITLEAKS_VERSION}_checksums.txt" + curl -fsSLO "${base}/${archive}" + curl -fsSLO "${base}/${checksums}" + grep " ${archive}$" "${checksums}" | sha256sum -c - + tar -xzf "${archive}" + sudo install -m 0755 gitleaks /usr/local/bin/gitleaks + - run: gitleaks git --no-banner --redact .