Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,16 @@ Affected Verification deterministically selects the smallest verification worklo

The operative claim is **minimum defensible verification**, not mathematical global minimality. Unknown impact is never permission to skip work.

This repository contains a dependency-free Node.js 20 prototype. It consumes normalized change, impact, check-level dependency-completeness, verification-catalog, and policy data and emits an `opsle.affected-verification.plan.v2` argument containing selected checks, skipped checks, boundary evidence, exact reasons, provenance hashes, uncertainty, escalation, and sufficiency. It plans work; it does not run CI.
This repository contains a dependency-free Node.js 20 planner. It consumes normalized change, impact, check-level dependency-completeness, verification-catalog, and policy data and emits an `opsle.affected-verification.plan.v2` argument containing selected checks, skipped checks, boundary evidence, exact reasons, provenance hashes, uncertainty, escalation, and sufficiency. It plans work; it does not run commands.

`opsle/tasks` is a production consumer of this public plan contract. Tasks captures
the staged BUILD tree, derives normalized input from a base-revision manifest,
invokes the CLI without a shell, validates the returned action IDs and commands
against that immutable input, and retains execution authority. AV output never
creates an unrestricted command surface. An empty change set and an empty
verification catalog are valid inputs so consumers can represent unchanged
builds and repositories with no automated checks without inventing work; the
result remains subject to the ordinary sufficiency and uncertainty rules.

## Try it

Expand Down Expand Up @@ -36,6 +45,14 @@ npm run verify

## Status

This is a narrow research prototype, not a trusted replacement for full verification. AV-EXP-001 observed no AV miss in its frozen JavaScript corpus; AV-EXP-002 permanently observed one AV miss in its frozen Python corpus; AV-EXP-003 selected that known check under the repair and observed zero repaired misses in its generalized and frozen replay corpora. All remain `SHADOW`, do not establish general safety, and do not provide production adapters.
AV's research evidence remains narrow and does not establish general selector
completeness. AV-EXP-001 observed no AV miss in its frozen JavaScript corpus;
AV-EXP-002 permanently observed one AV miss in its frozen Python corpus;
AV-EXP-003 selected that known check under the repair and observed zero repaired
misses in its generalized and frozen replay corpora. Tasks therefore treats
targeted selection as authoritative only when the repository manifest declares
complete impact, catalog, and check-boundary evidence. Unknown or incomplete
evidence broadens to the full configured command or stops; it never silently
becomes a passing verification result.

Apache-2.0.
11 changes: 7 additions & 4 deletions bin/affected-verification.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
canonicalJson,
classifyShadow,
operatorIndicator,
planTaskVerification,
planVerification,
} from '../src/index.js';
import { scenarios } from '../fixtures/scenarios.js';
Expand All @@ -34,6 +35,7 @@ function loadFixture(id) {
function usage() {
return [
'affected-verification plan <input.json> [--receipt <receipt.json>]',
'affected-verification task-plan <request.json> [--receipt <receipt.json>]',
'affected-verification fixture <id> [--receipt <receipt.json>]',
'affected-verification shadow <plan.json> <full-run.json>',
].join('\n');
Expand All @@ -43,13 +45,14 @@ try {
const [command, ...args] = process.argv.slice(2);
if (command === '--help' || command === '-h' || !command) {
process.stdout.write(`${usage()}\n`);
} else if (command === 'plan' || command === 'fixture') {
} else if (command === 'plan' || command === 'fixture' || command === 'task-plan') {
if (!args[0]) throw new InputError([`${command} requires an input`]);
const input = command === 'plan' ? readJson(args[0]) : loadFixture(args[0]);
const plan = planVerification(input);
const input = command === 'fixture' ? loadFixture(args[0]) : readJson(args[0]);
const taskPlan = command === 'task-plan' ? planTaskVerification(input) : null;
const plan = taskPlan?.plan ?? planVerification(input);
const receiptPath = parseReceipt(args);
if (receiptPath) writeFileSync(receiptPath, `${canonicalJson(buildValueReceipt(plan))}\n`);
process.stdout.write(`${canonicalJson(plan)}\n`);
process.stdout.write(`${canonicalJson(taskPlan ?? plan)}\n`);
process.stderr.write(`${operatorIndicator(plan)}\n`);
} else if (command === 'shadow') {
if (!args[0] || !args[1]) throw new InputError(['shadow requires a plan and full-run input']);
Expand Down
2 changes: 1 addition & 1 deletion schemas/plan-v2.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"additionalProperties": false,
"required": ["checks_assessed", "forced_check_ids", "states", "agreement_implies_completeness"],
"properties": {
"checks_assessed": { "type": "integer", "minimum": 1 },
"checks_assessed": { "type": "integer", "minimum": 0 },
"forced_check_ids": { "type": "array", "items": { "type": "string", "minLength": 1 } },
"states": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
"agreement_implies_completeness": { "const": false }
Expand Down
6 changes: 6 additions & 0 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ export {
validateShadowBenchmarkResult,
} from './benchmark.js';
export { buildValueReceipt, operatorIndicator } from './value-receipt.js';
export {
TASK_MANIFEST_SCHEMA,
TASK_PLAN_SCHEMA,
TASK_REQUEST_SCHEMA,
planTaskVerification,
} from './task.js';
export {
INPUT_SCHEMA,
PLAN_SCHEMA,
Expand Down
2 changes: 1 addition & 1 deletion src/planner.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { InputError, PLAN_SCHEMA, validateInput } from './validate.js';

const ESCALATION_ORDER = { NONE: 0, BROADEN: 1, FULL: 2, INVALIDATE: 3 };

function globMatches(pattern, path) {
export function globMatches(pattern, path) {
const escaped = pattern
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
.replace(/\*\*/g, '\u0000')
Expand Down
202 changes: 202 additions & 0 deletions src/task.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
import { contentIdentity, deepFreeze } from './canonical.js';
import { globMatches, planVerification } from './planner.js';
import { InputError } from './validate.js';

export const TASK_REQUEST_SCHEMA = 'opsle.affected-verification.task-request.v1';
export const TASK_MANIFEST_SCHEMA = 'opsle.affected-verification.manifest.v1';
export const TASK_PLAN_SCHEMA = 'opsle.affected-verification.task-plan.v1';

function object(value, path, issues) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
issues.push(`${path} must be an object`);
return {};
}
return value;
}

function array(value, path, issues) {
if (!Array.isArray(value)) {
issues.push(`${path} must be an array`);
return [];
}
return value;
}

function text(value, path, issues) {
if (typeof value !== 'string' || !value.trim()) issues.push(`${path} must be a nonempty string`);
return value;
}

function exactKeys(value, allowed, path, issues) {
for (const key of Object.keys(value)) {
if (!allowed.includes(key)) issues.push(`${path}.${key} is not allowed`);
}
}

function taskInput(request) {
const issues = [];
const root = object(request, 'request', issues);
exactKeys(root, ['schema', 'task', 'repository', 'change', 'manifest'], 'request', issues);
if (root.schema !== TASK_REQUEST_SCHEMA) issues.push(`request.schema must be ${TASK_REQUEST_SCHEMA}`);

const task = object(root.task, 'request.task', issues);
exactKeys(task, ['id', 'execution_id'], 'request.task', issues);
text(task.id, 'request.task.id', issues);
text(task.execution_id, 'request.task.execution_id', issues);

const repository = object(root.repository, 'request.repository', issues);
exactKeys(repository, ['identity', 'base_revision', 'target_revision'], 'request.repository', issues);
text(repository.identity, 'request.repository.identity', issues);
text(repository.base_revision, 'request.repository.base_revision', issues);
text(repository.target_revision, 'request.repository.target_revision', issues);

const change = object(root.change, 'request.change', issues);
exactKeys(change, ['identity', 'paths'], 'request.change', issues);
text(change.identity, 'request.change.identity', issues);
const paths = array(change.paths, 'request.change.paths', issues);
for (const [index, itemValue] of paths.entries()) {
const item = object(itemValue, `request.change.paths[${index}]`, issues);
exactKeys(item, ['path', 'regions', 'risk_tags'], `request.change.paths[${index}]`, issues);
text(item.path, `request.change.paths[${index}].path`, issues);
array(item.regions ?? [], `request.change.paths[${index}].regions`, issues);
array(item.risk_tags ?? [], `request.change.paths[${index}].risk_tags`, issues);
}

const manifest = object(root.manifest, 'request.manifest', issues);
exactKeys(manifest, [
'schema', 'source_path', 'source_identity', 'evidence_complete',
'catalog_complete', 'components', 'checks', 'policy',
], 'request.manifest', issues);
if (manifest.schema !== TASK_MANIFEST_SCHEMA) {
issues.push(`request.manifest.schema must be ${TASK_MANIFEST_SCHEMA}`);
}
text(manifest.source_path, 'request.manifest.source_path', issues);
text(manifest.source_identity, 'request.manifest.source_identity', issues);
if (typeof manifest.evidence_complete !== 'boolean') {
issues.push('request.manifest.evidence_complete must be a boolean');
}
if (typeof manifest.catalog_complete !== 'boolean') {
issues.push('request.manifest.catalog_complete must be a boolean');
}
const components = array(manifest.components, 'request.manifest.components', issues);
for (const [index, componentValue] of components.entries()) {
const component = object(componentValue, `request.manifest.components[${index}]`, issues);
exactKeys(component, ['id', 'dependencies', 'path_globs', 'risk_tags'], `request.manifest.components[${index}]`, issues);
text(component.id, `request.manifest.components[${index}].id`, issues);
const globs = array(component.path_globs, `request.manifest.components[${index}].path_globs`, issues);
if (!globs.length) issues.push(`request.manifest.components[${index}].path_globs must not be empty`);
globs.forEach((glob, globIndex) => text(glob, `request.manifest.components[${index}].path_globs[${globIndex}]`, issues));
array(component.dependencies, `request.manifest.components[${index}].dependencies`, issues);
array(component.risk_tags ?? [], `request.manifest.components[${index}].risk_tags`, issues);
}
array(manifest.checks, 'request.manifest.checks', issues);
object(manifest.policy, 'request.manifest.policy', issues);
if (issues.length) throw new InputError(issues);
return root;
}

function assessment(check, providerId) {
const dependency = check.dependency ?? {
completeness: 'UNKNOWN',
mechanisms: [{ kind: 'DECLARED_SCOPE', positive: false }],
boundaries: [],
explanation: 'The manifest did not declare complete check-level dependency evidence.',
};
return {
check_id: check.id,
completeness: dependency.completeness,
mechanisms: dependency.mechanisms.map((item) => ({
kind: item.kind,
positive: item.positive,
evidence_refs: [providerId],
})),
boundaries: dependency.boundaries.map((item) => ({
...item,
evidence_refs: [providerId],
})),
explanation: dependency.explanation,
};
}

export function planTaskVerification(rawRequest) {
const request = taskInput(rawRequest);
const providerId = 'base-revision-manifest';
const impacts = request.change.paths.map((changed) => {
const matches = request.manifest.components.filter((component) =>
component.path_globs.some((glob) => globMatches(glob, changed.path)));
return {
path: changed.path,
components: [...new Set(matches.map((item) => item.id))].sort(),
confidence: matches.length ? 'KNOWN' : 'UNKNOWN',
reason: matches.length
? `Matched immutable manifest components: ${matches.map((item) => item.id).sort().join(', ')}`
: 'No immutable manifest component owns this changed path.',
};
});
const changedPaths = request.change.paths.map((changed) => {
const matchedTags = request.manifest.components
.filter((component) => component.path_globs.some((glob) => globMatches(glob, changed.path)))
.flatMap((component) => component.risk_tags ?? []);
return {
path: changed.path,
regions: [...new Set(changed.regions ?? [])].sort(),
risk_tags: [...new Set([...(changed.risk_tags ?? []), ...matchedTags])].sort(),
};
});
const input = {
schema: 'opsle.affected-verification.input.v2',
change: {
base_revision: request.repository.base_revision,
target_revision: request.repository.target_revision,
identity: request.change.identity,
paths: changedPaths,
},
evidence: {
identity: contentIdentity({
manifest: request.manifest.source_identity,
paths: changedPaths,
impacts,
}),
complete: request.manifest.evidence_complete,
providers: [{
id: providerId,
kind: 'BASE_REVISION_MANIFEST',
version: '1',
identity: request.manifest.source_identity,
}],
components: request.manifest.components.map((component) => ({
id: component.id,
dependencies: component.dependencies,
})),
impacts,
check_dependencies: request.manifest.checks.map((check) => assessment(check, providerId)),
},
catalog: {
identity: contentIdentity(request.manifest.checks),
complete: request.manifest.catalog_complete,
checks: request.manifest.checks.map(({ dependency: _dependency, ...check }) => check),
},
policy: {
identity: contentIdentity(request.manifest.policy),
...request.manifest.policy,
},
};
const plan = planVerification(input);
const result = {
schema: TASK_PLAN_SCHEMA,
decision_identity: null,
task: { ...request.task },
repository: { ...request.repository },
manifest: {
schema: request.manifest.schema,
source_path: request.manifest.source_path,
source_identity: request.manifest.source_identity,
},
impacts,
plan,
};
return deepFreeze({
...result,
decision_identity: contentIdentity({ ...result, decision_identity: undefined }),
});
}
6 changes: 3 additions & 3 deletions src/validate.js
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ function validateChange(raw, issues) {
stringAt(change.base_revision, 'change.base_revision', issues);
stringAt(change.target_revision, 'change.target_revision', issues);
if (change.identity !== undefined) stringAt(change.identity, 'change.identity', issues);
const paths = arrayAt(change.paths, 'change.paths', issues, { nonempty: true });
const paths = arrayAt(change.paths, 'change.paths', issues);
paths.forEach((rawPath, index) => {
const path = objectAt(rawPath, `change.paths[${index}]`, issues);
rejectUnknownKeys(path, ['path', 'regions', 'risk_tags'], `change.paths[${index}]`, issues);
Expand Down Expand Up @@ -201,7 +201,7 @@ function validateCheckDependencies(evidence, catalog, issues) {
evidence.check_dependencies,
'evidence.check_dependencies',
issues,
{ nonempty: true },
{ nonempty: (catalog.checks ?? []).length > 0 },
);
assessments.forEach((rawAssessment, index) => {
const path = `evidence.check_dependencies[${index}]`;
Expand Down Expand Up @@ -308,7 +308,7 @@ function validateCatalog(raw, componentIds, issues) {
rejectUnknownKeys(catalog, ['identity', 'complete', 'checks'], 'catalog', issues);
stringAt(catalog.identity, 'catalog.identity', issues);
if (typeof catalog.complete !== 'boolean') issues.push('catalog.complete must be a boolean');
const checks = arrayAt(catalog.checks, 'catalog.checks', issues, { nonempty: true });
const checks = arrayAt(catalog.checks, 'catalog.checks', issues);
checks.forEach((rawCheck, index) => {
const check = objectAt(rawCheck, `catalog.checks[${index}]`, issues);
rejectUnknownKeys(check, ['id', 'type', 'command', 'scope', 'tags', 'test_executions', 'cost'], `catalog.checks[${index}]`, issues);
Expand Down
22 changes: 22 additions & 0 deletions tests/planner.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,28 @@ test('identical input produces an identical deterministic plan', () => {
assert.deepEqual(planVerification(input), planVerification(structuredClone(input)));
});

test('an unchanged tree is representable and can conservatively select the full catalog', () => {
const input = fixture('isolated-implementation');
input.change.paths = [];
input.evidence.impacts = [];
input.evidence.complete = false;
const plan = planVerification(input);
assert.equal(plan.sufficiency, 'FULL_VERIFICATION_REQUIRED');
assert.equal(plan.change.changed_paths.length, 0);
assert.equal(plan.skipped_checks.length, 0);
});

test('a repository with no automated checks returns explicit insufficient evidence', () => {
const input = fixture('readme-only');
input.catalog.checks = [];
input.evidence.check_dependencies = [];
const plan = planVerification(input);
assert.equal(plan.sufficiency, 'INSUFFICIENT_EVIDENCE');
assert.deepEqual(plan.selected_checks, []);
assert.deepEqual(plan.skipped_checks, []);
assert.ok(plan.uncertainty.reasons.includes('NO_VERIFICATION_COVERAGE:docs'));
});

test('unrelated checks are skipped with exact reasons', () => {
const plan = planVerification(fixture('isolated-implementation'));
const skipped = plan.skipped_checks.find((item) => item.id === 'unrelated.large-suite');
Expand Down
Loading
Loading