diff --git a/.github/workflows/v2-domain.yml b/.github/workflows/v2-domain.yml new file mode 100644 index 0000000..0e62063 --- /dev/null +++ b/.github/workflows/v2-domain.yml @@ -0,0 +1,27 @@ +name: V2 domain contracts +on: + pull_request: + paths: + - 'shared/v2/**' + - 'server/v2/**' + - 'tests/v2/**' + - 'tsconfig.v2.json' + - 'package*.json' + - '.github/workflows/v2-domain.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + domain: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci --ignore-scripts + - run: npm run typecheck:v2 + - run: npm run test:v2 + - run: npm run build diff --git a/.gitignore b/.gitignore index 3b91ac9..f6e81b0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ lerna-debug.log* node_modules dist dist-ssr +.v2-build *.local # Editor directories and files @@ -27,4 +28,4 @@ dist-ssr *.sln *.sw? -.env \ No newline at end of file +.env diff --git a/docs/v2/README.md b/docs/v2/README.md new file mode 100644 index 0000000..55daa54 --- /dev/null +++ b/docs/v2/README.md @@ -0,0 +1,36 @@ +# HubCyS V2 domain foundation + +This additive foundation establishes validated records and pure domain rules for evidence-backed readiness. It does not change live routes, legacy Firestore collections/rules, identity bootstrap, API authentication, billing, or production behavior. + +## Included + +- Strict V2 record and command contracts with tenant ownership, stable capability/threat identifiers, separate observations/evidence/findings, remediation, verification, retest, reporting, advisory and audit concepts. +- Deterministic exercise/capability scoring with versioned policy, published criterion manifests, retained inputs and input digest. Unknown or ineligible evidence withholds an unqualified score; advisory, legacy and simulated-artifact records cannot count as participant-performance evidence. +- Central command authorization policy for organization membership, exercise assignments, ownership, independent verification and explicitly scoped provider-customer grants. +- Exercise/remediation state invariant guards and comparable persistent-gap detection. Completion remains separate from verification and does not alter readiness. +- Isolated strict TypeScript compilation and Node tests without Firebase, provider calls, customer records or secrets. + +## Run + +```sh +npm ci --ignore-scripts +npm run typecheck:v2 +npm run test:v2 +npm run build +``` + +Compilation writes temporary test output to the ignored `.v2-build` directory. Existing root lint/typecheck remain separate and must not be represented as repaired by this change. + +## Integration boundary + +These modules are not exposed as browser APIs and are not wired into legacy workflows. An integration must verify the identity token, resolve memberships/grants/assignments from authoritative server storage, validate all parent-child references, and call the policy before privileged database operations. + +The scorer expects the entire published criterion and expected-action manifests, not a caller-selected successful subset. It returns `inputs` separately from aggregate `result`/`capabilities`; persist individual input records and immutable run/result records transactionally, rather than storing only a percentage or a growing JSON blob. + +`transitionRemediation` checks local state/evidence/verification invariants but does not establish that a retest executed successfully. A later verification command must validate the retest request, baseline-to-retest obligation mapping, accepted outcomes, comparability, complete finding obligations and verifier authorization before using this guard; never accept a Verification record supplied by a browser. + +`authorizeCommand` is an authorization primitive, not JWT verification. Its principal and record arguments are trusted server inputs only; no existing endpoint or Firestore rule becomes protected merely by importing this module. + +## Still required + +Server identity verification, authoritative membership provisioning, Firestore/Storage policies and emulator tests, transactional commands/audit/outbox, durable exercise runtime, real participant responses, file ingestion, complete verification/retest services, organization readiness projection, reports, migration tooling, and environment-matched end-to-end acceptance remain subsequent review units. No production-readiness claim is made by the unit suite. diff --git a/package.json b/package.json index e1f7668..1f20806 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,8 @@ "lint": "eslint . --quiet", "lint:fix": "eslint . --fix", "typecheck": "tsc -p ./jsconfig.json", + "typecheck:v2": "tsc -p tsconfig.v2.json --noEmit", + "test:v2": "tsc -p tsconfig.v2.json && node --test .v2-build/tests/v2/*.test.js", "preview": "vite preview" }, "dependencies": { diff --git a/server/v2/authorization.ts b/server/v2/authorization.ts new file mode 100644 index 0000000..7f1e90a --- /dev/null +++ b/server/v2/authorization.ts @@ -0,0 +1,88 @@ +import { z } from 'zod'; +import { + Id, Instant, Membership, Permission, ProviderGrant, ExerciseParticipant, + ReportClass, type PermissionName, +} from '../../shared/v2/contracts.js'; + +const permissionSets: Record['roles'][number], readonly PermissionName[]> = { + organization_admin: ['organization:manage', 'exercise:create', 'exercise:start', 'inject:release', 'response:submit', + 'observation:accept', 'remediation:assign', 'remediation:update', 'verification:record', 'report:read', 'report:export'], + readiness_lead: ['exercise:create', 'exercise:start', 'inject:release', 'response:submit', 'observation:accept', + 'remediation:assign', 'remediation:update', 'verification:record', 'report:read', 'report:export'], + facilitator: ['exercise:start', 'inject:release', 'response:submit', 'remediation:update', 'report:read'], + evaluator: ['observation:accept', 'verification:record', 'response:submit', 'remediation:update', 'report:read'], + participant: ['response:submit', 'remediation:update'], + observer: ['report:read'], + auditor: ['report:read', 'report:export'], +}; + +const AuthorizationRequest = z.object({ + // The caller of this library MUST derive principal_uid from a verified token. + principal_uid: Id, + organization_id: Id, + organization_status: z.enum(['active', 'suspended']), + permission: Permission, + now: Instant, + membership: Membership, + provider_grant: ProviderGrant.optional(), + exercise_id: Id.optional(), + exercise_participant: ExerciseParticipant.optional(), + action_owner_uid: Id.optional(), + verification_subject_owner_uid: Id.optional(), + report_class: ReportClass.optional(), +}).strict(); + +export class AuthorizationError extends Error { + constructor() { super('FORBIDDEN'); this.name = 'AuthorizationError'; } +} + +/** + * Central policy primitive, not token authentication. Inputs must be loaded + * server-side. Never accept membership/grant/assignment objects from a browser. + */ +export function authorizeCommand(raw: unknown): void { + const parsed = AuthorizationRequest.safeParse(raw); + if (!parsed.success) throw new AuthorizationError(); + const request = parsed.data; + const { membership, permission, principal_uid, organization_id, provider_grant: grant } = request; + const deny = () => { throw new AuthorizationError(); }; + if (request.organization_status !== 'active' || membership.status !== 'active' || membership.uid !== principal_uid) deny(); + if (membership.organization_id !== organization_id) { + if (!grant || grant.status !== 'active' || grant.organization_id !== organization_id || + grant.provider_organization_id !== membership.organization_id || grant.subject_uid !== principal_uid || + Date.parse(grant.expires_at) <= Date.parse(request.now) || !grant.permissions.includes(permission)) deny(); + } else if (grant) { + // A customer-local member must not smuggle an unrelated provider grant. + deny(); + } + if (!membership.roles.some(role => permissionSets[role].includes(permission))) deny(); + + const needsAssignment = ['exercise:start', 'inject:release', 'response:submit', 'observation:accept'].includes(permission); + if (needsAssignment) { + const assignment = request.exercise_participant; + if (!request.exercise_id || !assignment || assignment.exercise_id !== request.exercise_id || + assignment.organization_id !== organization_id || assignment.uid !== principal_uid || assignment.status !== 'active') deny(); + const requiredRole = permission === 'response:submit' ? 'participant' + : permission === 'observation:accept' ? 'evaluator' : 'facilitator'; + if (!assignment?.roles.includes(requiredRole)) deny(); + } + if (permission === 'remediation:update') { + const managesWork = membership.roles.includes('organization_admin') || membership.roles.includes('readiness_lead'); + if (!managesWork && request.action_owner_uid !== principal_uid) deny(); + } + if (permission === 'verification:record') { + if (!request.verification_subject_owner_uid || request.verification_subject_owner_uid === principal_uid) deny(); + } + if (permission === 'report:read' || permission === 'report:export') { + if (!request.report_class) deny(); + const broadReportAccess = membership.roles.some(role => + ['organization_admin', 'readiness_lead', 'auditor'].includes(role)); + if (!broadReportAccess) { + if (membership.roles.includes('observer') && request.report_class === 'executive_readiness') return; + const assignment = request.exercise_participant; + if (request.report_class !== 'exercise_after_action' || !request.exercise_id || !assignment || + assignment.organization_id !== organization_id || assignment.exercise_id !== request.exercise_id || + assignment.uid !== principal_uid || assignment.status !== 'active') deny(); + } + } +} diff --git a/server/v2/lifecycle.ts b/server/v2/lifecycle.ts new file mode 100644 index 0000000..8ee4b2a --- /dev/null +++ b/server/v2/lifecycle.ts @@ -0,0 +1,142 @@ +import { z } from 'zod'; +import { + ActionState, Evidence, ExerciseState, Id, Instant, RemediationAction, Verification, + assertTenant, uniqueById, +} from '../../shared/v2/contracts.js'; + +const exerciseTransitions: Record, readonly z.infer[]> = { + draft: ['scheduled', 'cancelled'], + scheduled: ['ready', 'cancelled'], + ready: ['running', 'cancelled'], + running: ['paused', 'review', 'aborted'], + paused: ['running', 'review', 'aborted'], + aborted: ['review'], + review: ['completed'], + completed: ['archived'], + archived: [], + cancelled: [], +}; + +export function assertExerciseTransition(from: unknown, to: unknown): void { + const current = ExerciseState.parse(from); + const next = ExerciseState.parse(to); + if (!exerciseTransitions[current].includes(next)) throw new Error('INVALID_EXERCISE_TRANSITION'); +} + +const actionTransitions: Record, readonly z.infer[]> = { + open: ['assigned'], + assigned: ['in_progress'], + in_progress: ['blocked', 'ready_for_verification'], + blocked: ['in_progress'], + ready_for_verification: ['verified', 'failed_verification'], + failed_verification: ['in_progress'], + verified: ['closed'], + closed: [], +}; +const verificationEvidenceKinds = new Set(['participant_response', 'facilitator_observation', 'system_event', 'file']); +const TransitionRequest = z.object({ + action: RemediationAction, + next_state: ActionState, + expected_record_version: z.number().int().nonnegative(), + now: Instant, + actor_uid: Id, + evidence: z.array(Evidence), + verification: Verification.optional(), +}).strict(); + +/** + * State invariant guard only. Caller must authorize the command and, for a + * targeted retest, independently validate the linked retest outcome. This + * function neither creates verification records nor changes readiness scores. + */ +export function transitionRemediation(raw: unknown) { + const request = TransitionRequest.parse(raw); + const { action, next_state, verification, evidence } = request; + assertTenant(action.organization_id, [...evidence, ...(verification ? [verification] : [])]); + if (action.record_version !== request.expected_record_version) throw new Error('RECORD_VERSION_CONFLICT'); + if (!actionTransitions[action.state].includes(next_state)) throw new Error('INVALID_REMEDIATION_TRANSITION'); + if (!action.owner_uid) throw new Error('ACTION_OWNER_REQUIRED'); + const evidenceMap = uniqueById(evidence); + const validEvidence = (ids: string[]) => ids.length > 0 && ids.every(id => { + const item = evidenceMap.get(id); + return item && item.status === 'accepted' && verificationEvidenceKinds.has(item.source_kind); + }); + if (next_state === 'ready_for_verification' && !validEvidence(action.completion_evidence_ids)) { + throw new Error('COMPLETION_EVIDENCE_REQUIRED'); + } + if (['verified', 'failed_verification', 'closed'].includes(next_state)) { + if (!verification || verification.action_id !== action.id || + verification.verifier_uid === action.owner_uid || + verification.verifier_uid !== request.actor_uid || + verification.method !== action.verification_method || + verification.finding_ids.length !== action.finding_ids.length || + !action.finding_ids.every(id => verification.finding_ids.includes(id)) || + !validEvidence(verification.evidence_ids)) throw new Error('VALID_INDEPENDENT_VERIFICATION_REQUIRED'); + if (Date.parse(verification.completed_at) > Date.parse(request.now)) throw new Error('VERIFICATION_IN_FUTURE'); + if (next_state === 'failed_verification' && verification.result !== 'fail') throw new Error('VERIFICATION_RESULT_MISMATCH'); + if (next_state !== 'failed_verification' && verification.result !== 'pass') throw new Error('VERIFICATION_RESULT_MISMATCH'); + if (verification.method === 'targeted_retest' && + verification.evidence_ids.some(id => evidenceMap.get(id)?.exercise_id !== verification.retest_exercise_id)) { + throw new Error('RETEST_EVIDENCE_SCOPE_MISMATCH'); + } + } + return { + action: { + ...action, + state: next_state, + record_version: action.record_version + 1, + }, + // Return an event description; persistence must append it transactionally. + transition: { + actor_uid: request.actor_uid, + occurred_at: request.now, + from: action.state, + to: next_state, + verification_id: verification?.id ?? null, + }, + }; +} + +const Occurrence = z.object({ + exercise_id: Id, + organization_id: Id, + capability_id: Id, + obligation_key: Id, + scope_key: Id, + context_version_id: Id, + scoring_policy_version: Id, + finalized_at: Instant, + outcome: z.enum(['pass', 'fail', 'unknown']), +}).strict(); + +/** Caller selects one comparable obligation cohort; never aggregate dissimilar scopes. */ +export function persistentGap(raw: unknown) { + const records = z.array(Occurrence).parse(raw); + const ids = new Set(); + const cohort = (record: z.infer) => JSON.stringify([ + record.organization_id, record.capability_id, record.obligation_key, + record.scope_key, record.context_version_id, record.scoring_policy_version, + ]); + const first = records[0]; + for (const record of records) { + if (ids.has(record.exercise_id)) throw new Error('DUPLICATE_EXERCISE_OCCURRENCE'); + ids.add(record.exercise_id); + if (first && cohort(record) !== cohort(first)) throw new Error('NON_COMPARABLE_OCCURRENCES'); + } + const ordered = [...records].sort((a, b) => { + const delta = Date.parse(a.finalized_at) - Date.parse(b.finalized_at); + return delta || a.exercise_id.localeCompare(b.exercise_id, 'en'); + }); + for (let i = 1; i < ordered.length; i++) { + const previous = ordered[i - 1]!; + const current = ordered[i]!; + if (Date.parse(previous.finalized_at) === Date.parse(current.finalized_at) && + previous.outcome !== current.outcome) throw new Error('AMBIGUOUS_OCCURRENCE_ORDER'); + } + const failures: string[] = []; + for (const record of ordered) { + if (record.outcome === 'pass') failures.length = 0; + else if (record.outcome === 'fail') failures.push(record.exercise_id); + } + return { persistent: failures.length >= 2, consecutive_failures: failures.length, exercise_ids: failures }; +} diff --git a/server/v2/scoring.ts b/server/v2/scoring.ts new file mode 100644 index 0000000..4c3f092 --- /dev/null +++ b/server/v2/scoring.ts @@ -0,0 +1,179 @@ +import { createHash } from 'node:crypto'; +import { z } from 'zod'; +import { + Criterion, Evidence, Exercise, ExpectedAction, Observation, ScenarioVersion, + assertTenant, uniqueById, +} from '../../shared/v2/contracts.js'; + +export const SCORING_POLICY_VERSION = 'readiness-v1'; +const scorableSources = new Set(['participant_response', 'facilitator_observation', 'system_event', 'file']); + +const ScoringRequest = z.object({ + exercise: Exercise, + scenario: ScenarioVersion, + expected_actions: z.array(ExpectedAction).min(1).max(500), + criteria: z.array(Criterion).min(1).max(500), + observations: z.array(Observation).max(500), + evidence: z.array(Evidence).max(2000), +}).strict(); + +type CriterionRecord = z.infer; +type ObservationRecord = z.infer; + +export type ScoringInput = { + criterion: CriterionRecord; + observation: ObservationRecord | null; + evidence: z.infer[]; + included: boolean; + points: number | null; + reason: string; +}; + +function pointsFor(criterion: CriterionRecord, observation: ObservationRecord): number | null { + const measurement = observation.measurement; + if (measurement.kind === 'unknown') return null; + if (criterion.kind !== measurement.kind) throw new Error('MEASUREMENT_KIND_MISMATCH'); + switch (measurement.kind) { + case 'completion': + return measurement.completed ? 1 : measurement.omission_observed ? 0 : null; + case 'deadline': + if (criterion.kind !== 'deadline') throw new Error('MEASUREMENT_KIND_MISMATCH'); + return measurement.elapsed_ms <= criterion.max_elapsed_ms ? 1 : 0; + case 'sequence': + return measurement.followed ? 1 : 0; + case 'quality': + return { fail: 0, partial: 0.5, pass: 1 }[measurement.rating]; + } +} + +function aggregate(inputs: ScoringInput[]) { + const required = inputs.filter(input => input.criterion.required); + const plannedWeight = required.reduce((sum, input) => sum + input.criterion.weight, 0); + const eligible = required.filter(input => input.included); + const evaluatedWeight = eligible.reduce((sum, input) => sum + input.criterion.weight, 0); + const earnedWeight = eligible.reduce((sum, input) => sum + input.criterion.weight * (input.points ?? 0), 0); + const provisionalScore = evaluatedWeight > 0 ? Math.round(100 * earnedWeight / evaluatedWeight) : null; + const complete = plannedWeight > 0 && evaluatedWeight === plannedWeight; + const observed = required.some(input => input.observation !== null); + return { + status: complete ? 'scored' : observed ? 'insufficient_evidence' : 'not_tested', + score: complete ? provisionalScore : null, + provisional_score: provisionalScore, + earned_weight: earnedWeight, + evaluated_weight: evaluatedWeight, + planned_weight: plannedWeight, + coverage_percent: plannedWeight > 0 ? 100 * evaluatedWeight / plannedWeight : 0, + unknown_criterion_ids: required.filter(input => !input.included).map(input => input.criterion.id), + }; +} + +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value !== null && typeof value === 'object') { + return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b, 'en')).map( + ([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`, + ).join(',')}}`; + } + const json = JSON.stringify(value); + if (json === undefined) throw new Error('UNSERIALIZABLE_SCORE_INPUT'); + return json; +} + +/** + * Pure domain calculation. Call ONLY with server-loaded records from an + * authorized command; this function is not a persistence or authorization API. + * Returns input records separately from results for auditable persistence. + */ +export function calculateExerciseScore(raw: unknown) { + const request = ScoringRequest.parse(raw); + const { exercise, scenario, expected_actions, criteria, observations, evidence } = request; + assertTenant(exercise.organization_id, [scenario, ...expected_actions, ...criteria, ...observations, ...evidence]); + if (!['review', 'completed'].includes(exercise.state)) throw new Error('EXERCISE_NOT_REVIEWABLE'); + if (scenario.state !== 'published') throw new Error('SCENARIO_NOT_PUBLISHED'); + if (exercise.scenario_version_id !== scenario.id || exercise.threat_id !== scenario.threat_id) { + throw new Error('SCENARIO_MISMATCH'); + } + if (exercise.scoring_policy_version !== SCORING_POLICY_VERSION || + scenario.scoring_policy_version !== SCORING_POLICY_VERSION) throw new Error('UNSUPPORTED_SCORING_POLICY'); + + const actionMap = uniqueById(expected_actions); + const criteriaMap = uniqueById(criteria); + const evidenceMap = uniqueById(evidence); + uniqueById(observations); + const manifestMatches = (ids: string[], records: Map) => + ids.length === records.size && ids.every(id => records.has(id)); + if (!manifestMatches(scenario.expected_action_ids, actionMap) || !manifestMatches(scenario.criterion_ids, criteriaMap)) { + throw new Error('SCENARIO_MANIFEST_MISMATCH'); + } + const observationMap = new Map(); + for (const action of expected_actions) { + if (action.scenario_version_id !== scenario.id || !scenario.capability_ids.includes(action.capability_id)) { + throw new Error('EXPECTED_ACTION_SCOPE_MISMATCH'); + } + } + for (const item of evidence) { + if (item.exercise_id !== exercise.id) throw new Error('EVIDENCE_EXERCISE_MISMATCH'); + } + for (const criterion of criteria) { + const action = actionMap.get(criterion.expected_action_id); + if (!action || action.scenario_version_id !== scenario.id || action.capability_id !== criterion.capability_id || + criterion.scenario_version_id !== scenario.id || !scenario.capability_ids.includes(criterion.capability_id)) { + throw new Error('CRITERION_SCOPE_MISMATCH'); + } + } + for (const capability of scenario.capability_ids) { + if (!criteria.some(criterion => criterion.capability_id === capability && criterion.required)) { + throw new Error('CAPABILITY_HAS_NO_REQUIRED_CRITERIA'); + } + } + for (const observation of observations) { + const criterion = criteriaMap.get(observation.criterion_id); + if (!criterion || observation.exercise_id !== exercise.id || + observation.expected_action_id !== criterion.expected_action_id || + observation.capability_id !== criterion.capability_id) throw new Error('OBSERVATION_SCOPE_MISMATCH'); + if (observation.status === 'superseded') throw new Error('SUPERSEDED_OBSERVATION'); + if (observationMap.has(observation.criterion_id)) throw new Error('AMBIGUOUS_OBSERVATION_REVISION'); + for (const id of observation.evidence_ids) { + if (!evidenceMap.has(id)) throw new Error('DANGLING_EVIDENCE_REFERENCE'); + } + observationMap.set(observation.criterion_id, observation); + } + + const inputs: ScoringInput[] = [...criteria].sort((a, b) => a.id.localeCompare(b.id, 'en')).map(criterion => { + const storedObservation = observationMap.get(criterion.id); + const observation = storedObservation + ? { ...storedObservation, evidence_ids: [...storedObservation.evidence_ids].sort() } : null; + const linkedEvidence = (observation?.evidence_ids ?? []).map(id => evidenceMap.get(id)!); + let reason = 'evaluated'; + let points: number | null = null; + if (!criterion.required) reason = 'optional_not_in_required_score'; + else if (!observation) reason = 'no_observation'; + else if (observation.status !== 'accepted') reason = 'observation_not_accepted'; + else if (!linkedEvidence.length) reason = 'missing_required_evidence'; + else if (linkedEvidence.some(item => item.status !== 'accepted' || !scorableSources.has(item.source_kind))) { + reason = 'ineligible_evidence'; + } else { + points = pointsFor(criterion, observation); + if (points === null) reason = observation.measurement.kind === 'unknown' ? 'unknown_measurement' : 'omission_not_observed'; + } + return { criterion, observation, evidence: linkedEvidence, included: points !== null, points, reason }; + }); + const capabilities = [...scenario.capability_ids].sort().map(capability_id => ({ + capability_id, + ...aggregate(inputs.filter(input => input.criterion.capability_id === capability_id)), + })); + const input_sha256 = createHash('sha256').update(canonicalJson({ + policy_version: SCORING_POLICY_VERSION, exercise, scenario, + expected_actions: [...expected_actions].sort((a, b) => a.id.localeCompare(b.id, 'en')), + inputs, + })).digest('hex'); + return { + organization_id: exercise.organization_id, + exercise_id: exercise.id, + policy_version: SCORING_POLICY_VERSION, + input_sha256, + inputs, + result: aggregate(inputs), + capabilities, + }; +} diff --git a/shared/v2/contracts.ts b/shared/v2/contracts.ts new file mode 100644 index 0000000..028e74d --- /dev/null +++ b/shared/v2/contracts.ts @@ -0,0 +1,295 @@ +import { z } from 'zod'; + +// These contracts are additive and are NOT the schema for legacy collections. +export const Id = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/); +export const Instant = z.string().datetime(); +export const Digest = z.string().regex(/^[a-f0-9]{64}$/); +export const NonEmptyText = z.string().trim().min(1).max(20_000); +const uniqueIds = z.array(Id).max(500).refine(ids => new Set(ids).size === ids.length, 'Duplicate reference'); + +export const CapabilityId = z.enum([ + 'detection', 'triage', 'escalation', 'incident_command', + 'identity_containment', 'endpoint_containment', 'network_containment', + 'cloud_containment', 'evidence_preservation', 'threat_investigation', + 'internal_communication', 'executive_communication', 'legal_coordination', + 'regulatory_response', 'third_party_coordination', 'business_continuity', + 'recovery', 'lessons_learned', +]); +export const ThreatId = z.enum([ + 'ransomware', 'business_email_compromise', 'oauth_device_code_compromise', + 'credential_compromise', 'privileged_account_compromise', 'identity_compromise', + 'insider_threat', 'data_exfiltration', 'cloud_compromise', + 'third_party_compromise', 'supply_chain_compromise', 'destructive_malware', +]); +export const Role = z.enum([ + 'organization_admin', 'readiness_lead', 'facilitator', 'evaluator', + 'participant', 'observer', 'auditor', +]); +export const Permission = z.enum([ + 'organization:manage', 'exercise:create', 'exercise:start', 'inject:release', + 'response:submit', 'observation:accept', 'remediation:assign', + 'remediation:update', 'verification:record', 'report:read', 'report:export', +]); +export type PermissionName = z.infer; + +export const TenantRecord = z.object({ + id: Id, + organization_id: Id, + schema_version: z.literal(1), + created_at: Instant, + created_by_uid: Id, +}).strict(); + +export const Membership = TenantRecord.extend({ + uid: Id, + roles: z.array(Role).min(1).max(7).refine(r => new Set(r).size === r.length, 'Duplicate role'), + status: z.enum(['active', 'suspended', 'revoked']), +}).strict(); + +export const ProviderGrant = TenantRecord.extend({ + provider_organization_id: Id, + subject_uid: Id, + permissions: z.array(Permission).min(1), + status: z.enum(['active', 'revoked']), + expires_at: Instant, +}).strict(); + +export const ExerciseRole = z.enum(['facilitator', 'participant', 'observer', 'evaluator']); +export const ExerciseParticipant = TenantRecord.extend({ + exercise_id: Id, + uid: Id, + roles: z.array(ExerciseRole).min(1).max(4), + status: z.enum(['invited', 'active', 'withdrawn']), +}).strict(); + +export const ExerciseState = z.enum([ + 'draft', 'scheduled', 'ready', 'running', 'paused', 'review', + 'completed', 'archived', 'cancelled', 'aborted', +]); +export const Exercise = TenantRecord.extend({ + threat_id: ThreatId, + scenario_version_id: Id, + context_version_id: Id, + scoring_policy_version: Id, + state: ExerciseState, + purpose: z.enum(['baseline', 'exercise', 'retest']), + scope_key: Id, + record_version: z.number().int().nonnegative(), +}).strict(); + +export const ScenarioVersion = TenantRecord.extend({ + scenario_id: Id, + version: z.number().int().positive(), + state: z.enum(['draft', 'published', 'retired']), + threat_id: ThreatId, + capability_ids: z.array(CapabilityId).min(1).refine( + ids => new Set(ids).size === ids.length, 'Duplicate capability', + ), + expected_action_ids: uniqueIds.refine(ids => ids.length > 0), + criterion_ids: uniqueIds.refine(ids => ids.length > 0), + scoring_policy_version: Id, +}).strict(); + +export const ExpectedAction = TenantRecord.extend({ + scenario_version_id: Id, + capability_id: CapabilityId, + requirement: NonEmptyText, + policy_version_id: Id.nullable(), +}).strict(); + +const criterionBase = TenantRecord.extend({ + scenario_version_id: Id, + expected_action_id: Id, + capability_id: CapabilityId, + weight: z.number().int().min(1).max(100), + required: z.boolean(), + evidence_required: z.literal(true), +}); +export const Criterion = z.discriminatedUnion('kind', [ + criterionBase.extend({ kind: z.literal('completion') }).strict(), + criterionBase.extend({ + kind: z.literal('deadline'), max_elapsed_ms: z.number().int().nonnegative().max(604_800_000), + }).strict(), + criterionBase.extend({ kind: z.literal('sequence') }).strict(), + criterionBase.extend({ + kind: z.literal('quality'), + anchors: z.object({ fail: NonEmptyText, partial: NonEmptyText, pass: NonEmptyText }).strict(), + }).strict(), +]); + +export const Measurement = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('completion'), completed: z.boolean(), omission_observed: z.boolean(), + }).strict(), + z.object({ + kind: z.literal('deadline'), elapsed_ms: z.number().int().nonnegative().max(604_800_000), + }).strict(), + z.object({ kind: z.literal('sequence'), followed: z.boolean() }).strict(), + z.object({ + kind: z.literal('quality'), rating: z.enum(['fail', 'partial', 'pass']), + }).strict(), + z.object({ kind: z.literal('unknown'), reason: NonEmptyText }).strict(), +]); + +export const Evidence = TenantRecord.extend({ + exercise_id: Id, + source_kind: z.enum([ + 'participant_response', 'facilitator_observation', 'system_event', + 'file', 'legacy_import', 'ai_advisory', 'simulation_artifact', + ]), + source_record_id: Id, + content_sha256: Digest, + status: z.enum(['accepted', 'pending', 'quarantined', 'unavailable']), +}).strict(); + +export const Observation = TenantRecord.extend({ + exercise_id: Id, + criterion_id: Id, + expected_action_id: Id, + capability_id: CapabilityId, + status: z.enum(['draft', 'accepted', 'superseded']), + accepted_by_uid: Id.nullable(), + accepted_at: Instant.nullable(), + measurement: Measurement, + rationale: NonEmptyText, + evidence_ids: uniqueIds, +}).strict().superRefine((record, ctx) => { + if (record.status === 'accepted' && (!record.accepted_at || !record.accepted_by_uid)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'Accepted observation requires reviewer and time' }); + } + if (record.accepted_at && Date.parse(record.accepted_at) < Date.parse(record.created_at)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'Acceptance cannot precede observation creation' }); + } +}); + +export const InjectRelease = TenantRecord.extend({ + exercise_id: Id, + inject_definition_id: Id, + released_at: Instant, + sequence: z.number().int().nonnegative(), +}).strict(); + +export const Response = TenantRecord.extend({ + exercise_id: Id, + inject_release_id: Id, + actor_uid: Id, + received_at: Instant, + content: NonEmptyText, + receipt_id: Id, +}).strict(); + +export const Finding = TenantRecord.extend({ + exercise_id: Id, + threat_id: ThreatId, + capability_id: CapabilityId, + scope_key: Id, + observation_ids: uniqueIds.refine(ids => ids.length > 0, 'Finding requires observation'), + severity: z.enum(['low', 'medium', 'high', 'critical']), + state: z.enum(['open', 'under_remediation', 'awaiting_verification', 'verified', 'closed', 'risk_accepted']), +}).strict(); + +export const ActionState = z.enum([ + 'open', 'assigned', 'in_progress', 'blocked', 'ready_for_verification', + 'verified', 'failed_verification', 'closed', +]); +export const RemediationAction = TenantRecord.extend({ + finding_ids: uniqueIds.refine(ids => ids.length > 0, 'Action requires finding'), + owner_uid: Id.nullable(), + state: ActionState, + completion_evidence_ids: uniqueIds, + due_at: Instant.nullable(), + verification_requirements: NonEmptyText, + verification_method: z.enum(['document_review', 'targeted_retest']), + record_version: z.number().int().nonnegative(), +}).strict(); + +export const RetestRequest = TenantRecord.extend({ + baseline_exercise_id: Id, + retest_exercise_id: Id, + finding_ids: uniqueIds.refine(ids => ids.length > 0), + action_ids: uniqueIds.refine(ids => ids.length > 0), + targets: z.array(z.object({ + baseline_criterion_id: Id, + retest_criterion_id: Id, + capability_id: CapabilityId, + }).strict()).min(1).max(500), + scope_key: Id, + context_version_id: Id, +}).strict(); + +export const Verification = TenantRecord.extend({ + action_id: Id, + finding_ids: uniqueIds.refine(ids => ids.length > 0), + verifier_uid: Id, + method: z.enum(['document_review', 'targeted_retest']), + retest_exercise_id: Id.nullable(), + result: z.enum(['pass', 'fail', 'inconclusive']), + evidence_ids: uniqueIds.refine(ids => ids.length > 0), + completed_at: Instant, +}).strict().superRefine((record, ctx) => { + if (record.method === 'targeted_retest' && !record.retest_exercise_id) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'Capability verification requires retest' }); + } +}); + +export const ReportClass = z.enum(['executive_readiness', 'exercise_after_action', 'operational_readiness', 'audit_evidence']); +export const ReportSnapshot = TenantRecord.extend({ + report_class: ReportClass, + as_of: Instant, + score_run_ids: uniqueIds, + finding_ids: uniqueIds, + action_ids: uniqueIds, + verification_ids: uniqueIds, + evidence_ids: uniqueIds, + manifest_sha256: Digest, +}).strict(); + +export const AdvisoryOutput = TenantRecord.extend({ + kind: z.enum(['scenario', 'inject', 'summary', 'finding_suggestion', 'remediation_suggestion', 'retest_recommendation']), + model_id: NonEmptyText, + prompt_version: Id, + input_reference_ids: uniqueIds, + content: NonEmptyText, + review_state: z.enum(['unreviewed', 'accepted', 'rejected']), +}).strict(); + +export const AuditEvent = TenantRecord.extend({ + actor_uid: Id, + action: Id, + entity_type: Id, + entity_id: Id, + request_id: Id, + before_sha256: Digest.nullable(), + after_sha256: Digest, +}).strict(); + +// Parse request payloads separately from stored envelopes. Caller cannot set +// organization, identity, timestamps, score, reviewer, or verification result. +export const SubmitResponseCommand = z.object({ + exercise_id: Id, + inject_release_id: Id, + content: NonEmptyText, + idempotency_key: Id, +}).strict(); +export const CompleteRemediationCommand = z.object({ + action_id: Id, + expected_record_version: z.number().int().nonnegative(), + evidence_ids: uniqueIds.refine(ids => ids.length > 0), + idempotency_key: Id, +}).strict(); + +export function assertTenant(organizationId: string, records: ReadonlyArray<{ organization_id: string }>): void { + Id.parse(organizationId); + if (records.some(record => record.organization_id !== organizationId)) { + throw new Error('TENANT_MISMATCH'); + } +} + +export function uniqueById(records: readonly T[]): Map { + const map = new Map(); + for (const record of records) { + if (map.has(record.id)) throw new Error('DUPLICATE_RECORD_ID'); + map.set(record.id, record); + } + return map; +} diff --git a/tests/v2/authorization.test.ts b/tests/v2/authorization.test.ts new file mode 100644 index 0000000..7729ebd --- /dev/null +++ b/tests/v2/authorization.test.ts @@ -0,0 +1,121 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { authorizeCommand } from '../../server/v2/authorization.js'; +import { SubmitResponseCommand, CompleteRemediationCommand } from '../../shared/v2/contracts.js'; +import { base, membership, NOW } from './fixtures.js'; + +function authFixture() { + return { + principal_uid: 'leader', organization_id: 'org-a', organization_status: 'active', + permission: 'exercise:create', now: NOW, membership: membership(), + }; +} +function participant(roles: string[] = ['participant']) { + return { ...base('participant-1'), exercise_id: 'exercise-1', uid: 'leader', roles, status: 'active' }; +} + +test('active readiness lead can create exercise', () => assert.doesNotThrow(() => authorizeCommand(authFixture()))); +for (const role of ['participant', 'observer', 'auditor'] as const) { + test(`${role} cannot create exercise`, () => { + assert.throws(() => authorizeCommand({ ...authFixture(), membership: membership([role]) }), /FORBIDDEN/); + }); +} +for (const status of ['suspended', 'revoked'] as const) { + test(`${status} membership is denied`, () => { + assert.throws(() => authorizeCommand({ ...authFixture(), membership: { ...membership(), status } }), /FORBIDDEN/); + }); +} +test('suspended organization is denied', () => { + assert.throws(() => authorizeCommand({ ...authFixture(), organization_status: 'suspended' }), /FORBIDDEN/); +}); +test('forged tenant and mismatched principal are denied', () => { + assert.throws(() => authorizeCommand({ ...authFixture(), organization_id: 'org-b' }), /FORBIDDEN/); + assert.throws(() => authorizeCommand({ ...authFixture(), principal_uid: 'attacker' }), /FORBIDDEN/); +}); +test('arbitrary legacy admin fields and permissions fail strict parsing', () => { + assert.throws(() => authorizeCommand({ + ...authFixture(), membership: { ...membership(['participant']), is_super_admin: true }, + }), /FORBIDDEN/); + assert.throws(() => authorizeCommand({ ...authFixture(), permission: '*' }), /FORBIDDEN/); +}); +test('even organization admin must have facilitator assignment to release injects', () => { + const request = { ...authFixture(), membership: membership(['organization_admin']), permission: 'inject:release', exercise_id: 'exercise-1' }; + assert.throws(() => authorizeCommand(request), /FORBIDDEN/); + assert.doesNotThrow(() => authorizeCommand({ ...request, exercise_participant: participant(['facilitator']) })); +}); +test('participant cannot release injects even with spoofed facilitator assignment alone', () => { + assert.throws(() => authorizeCommand({ + ...authFixture(), membership: membership(['participant']), permission: 'inject:release', + exercise_id: 'exercise-1', exercise_participant: participant(['facilitator']), + }), /FORBIDDEN/); +}); +test('response requires active enrollment in exact exercise under own identity', () => { + const request = { + ...authFixture(), permission: 'response:submit', membership: membership(['participant']), + exercise_id: 'exercise-1', exercise_participant: participant(), + }; + assert.doesNotThrow(() => authorizeCommand(request)); + for (const patch of [{ exercise_id: 'elsewhere' }, { uid: 'other' }, { organization_id: 'org-b' }, { status: 'withdrawn' }]) { + assert.throws(() => authorizeCommand({ ...request, exercise_participant: { ...participant(), ...patch } }), /FORBIDDEN/); + } +}); +test('observer cannot respond', () => { + assert.throws(() => authorizeCommand({ + ...authFixture(), permission: 'response:submit', membership: membership(['observer']), + exercise_id: 'exercise-1', exercise_participant: participant(['observer']), + }), /FORBIDDEN/); +}); +test('participant can update only owned action', () => { + const request = { ...authFixture(), permission: 'remediation:update', membership: membership(['participant']) }; + assert.throws(() => authorizeCommand(request), /FORBIDDEN/); + assert.throws(() => authorizeCommand({ ...request, action_owner_uid: 'someone-else' }), /FORBIDDEN/); + assert.doesNotThrow(() => authorizeCommand({ ...request, action_owner_uid: 'leader' })); +}); +test('verification requires independent owner even for organization admin', () => { + const request = { ...authFixture(), permission: 'verification:record', membership: membership(['organization_admin']) }; + assert.throws(() => authorizeCommand(request), /FORBIDDEN/); + assert.throws(() => authorizeCommand({ ...request, verification_subject_owner_uid: 'leader' }), /FORBIDDEN/); + assert.doesNotThrow(() => authorizeCommand({ ...request, verification_subject_owner_uid: 'someone-else' })); +}); +test('provider customer grant is required, scoped, active, expiring and permission-limited', () => { + const request = { ...authFixture(), membership: { ...membership(), organization_id: 'provider-a' } }; + const grant = { + ...base('grant-1'), provider_organization_id: 'provider-a', subject_uid: 'leader', + permissions: ['exercise:create'], status: 'active', expires_at: '2026-09-24T00:00:00.000Z', + }; + assert.throws(() => authorizeCommand(request), /FORBIDDEN/); + assert.doesNotThrow(() => authorizeCommand({ ...request, provider_grant: grant })); + for (const patch of [ + { organization_id: 'org-b' }, { provider_organization_id: 'provider-b' }, + { subject_uid: 'other' }, { status: 'revoked' }, { permissions: ['report:read'] }, + { expires_at: NOW }, + ]) { + assert.throws(() => authorizeCommand({ ...request, provider_grant: { ...grant, ...patch } }), /FORBIDDEN/); + } +}); +test('provider grant cannot amplify permissions beyond provider membership role', () => { + assert.throws(() => authorizeCommand({ + ...authFixture(), membership: { ...membership(['observer']), organization_id: 'provider-a' }, + provider_grant: { + ...base('grant-1'), provider_organization_id: 'provider-a', subject_uid: 'leader', + permissions: ['exercise:create'], status: 'active', expires_at: '2026-09-24T00:00:00.000Z', + }, + }), /FORBIDDEN/); +}); +test('reports require class scope and restrict observers to executive view', () => { + const request = { ...authFixture(), permission: 'report:read', membership: membership(['observer']) }; + assert.throws(() => authorizeCommand(request), /FORBIDDEN/); + assert.throws(() => authorizeCommand({ ...request, report_class: 'audit_evidence' }), /FORBIDDEN/); + assert.doesNotThrow(() => authorizeCommand({ ...request, report_class: 'executive_readiness' })); +}); +test('command payloads reject caller-owned identity, tenant and truth fields', () => { + const response = { exercise_id: 'ex-1', inject_release_id: 'release-1', content: 'Escalated', idempotency_key: 'request-1' }; + assert.doesNotThrow(() => SubmitResponseCommand.parse(response)); + for (const patch of [{ actor_uid: 'other' }, { organization_id: 'org-b' }, { received_at: NOW }, { score: 100 }]) { + assert.throws(() => SubmitResponseCommand.parse({ ...response, ...patch })); + } + assert.throws(() => CompleteRemediationCommand.parse({ + action_id: 'action-1', expected_record_version: 1, evidence_ids: ['evidence-1'], + idempotency_key: 'request-1', state: 'verified', + })); +}); diff --git a/tests/v2/fixtures.ts b/tests/v2/fixtures.ts new file mode 100644 index 0000000..aa307df --- /dev/null +++ b/tests/v2/fixtures.ts @@ -0,0 +1,72 @@ +import type { z } from 'zod'; +import { Evidence, Exercise, ScenarioVersion, ExpectedAction, Criterion, Observation, Membership, RemediationAction, Verification } from '../../shared/v2/contracts.js'; + +export const NOW = '2026-09-23T08:00:00.000Z'; +export function base(id: string) { + return { id, organization_id: 'org-a', schema_version: 1 as const, created_at: NOW, created_by_uid: 'leader' }; +} +export function scoringFixture() { + const exercise: z.infer = { + ...base('exercise-1'), threat_id: 'ransomware', scenario_version_id: 'scenario-v1', + context_version_id: 'context-v1', scoring_policy_version: 'readiness-v1', + state: 'review', purpose: 'baseline', scope_key: 'production', record_version: 1, + }; + const scenario: z.infer = { + ...base('scenario-v1'), scenario_id: 'ransomware-1', version: 1, state: 'published', + threat_id: 'ransomware', capability_ids: ['evidence_preservation'], + scoring_policy_version: 'readiness-v1', + expected_action_ids: ['expected-1', 'expected-2', 'expected-3', 'expected-4'], + criterion_ids: ['criterion-1', 'criterion-2', 'criterion-3', 'criterion-4'], + }; + const expected_actions: z.infer[] = [1, 2, 3, 4].map(i => ({ + ...base(`expected-${i}`), scenario_version_id: scenario.id, capability_id: 'evidence_preservation', + requirement: `Preservation requirement ${i}`, policy_version_id: null, + })); + const criterionBase = (i: number, weight: number) => ({ + ...base(`criterion-${i}`), scenario_version_id: scenario.id, expected_action_id: `expected-${i}`, + capability_id: 'evidence_preservation' as const, weight, required: true, evidence_required: true as const, + }); + const criteria: z.infer[] = [ + { ...criterionBase(1, 4), kind: 'completion' }, + { ...criterionBase(2, 3), kind: 'sequence' }, + { ...criterionBase(3, 2), kind: 'quality', anchors: { fail: 'Not preserved', partial: 'Partially preserved', pass: 'Fully preserved' } }, + { ...criterionBase(4, 1), kind: 'deadline', max_elapsed_ms: 900_000 }, + ]; + const evidence: z.infer[] = [1, 2, 3, 4].map(i => ({ + ...base(`evidence-${i}`), exercise_id: exercise.id, source_kind: 'participant_response', + source_record_id: `response-${i}`, content_sha256: 'a'.repeat(64), status: 'accepted', + })); + const measurements: z.infer['measurement'][] = [ + { kind: 'completion', completed: true, omission_observed: false }, + { kind: 'sequence', followed: false }, + { kind: 'quality', rating: 'partial' }, + { kind: 'deadline', elapsed_ms: 900_000 }, + ]; + const observations: z.infer[] = [1, 2, 3, 4].map(i => ({ + ...base(`observation-${i}`), exercise_id: exercise.id, criterion_id: `criterion-${i}`, + expected_action_id: `expected-${i}`, capability_id: 'evidence_preservation', + status: 'accepted', accepted_by_uid: 'evaluator', accepted_at: NOW, + measurement: measurements[i - 1]!, rationale: 'Reviewed against captured response', + evidence_ids: [`evidence-${i}`], + })); + return { exercise, scenario, expected_actions, criteria, evidence, observations }; +} + +export function membership(roles: z.infer['roles'] = ['readiness_lead']) { + return { ...base('membership-1'), uid: 'leader', roles, status: 'active' as const }; +} +export function actionFixture(): z.infer { + return { + ...base('action-1'), finding_ids: ['finding-1'], owner_uid: 'owner', + state: 'in_progress', completion_evidence_ids: ['evidence-1'], due_at: NOW, + verification_requirements: 'Preserve evidence in a targeted retest', + verification_method: 'targeted_retest', record_version: 1, + }; +} +export function verificationFixture(): z.infer { + return { + ...base('verification-1'), action_id: 'action-1', finding_ids: ['finding-1'], + verifier_uid: 'evaluator', method: 'targeted_retest', retest_exercise_id: 'exercise-1', + result: 'pass', evidence_ids: ['evidence-1'], completed_at: NOW, + }; +} diff --git a/tests/v2/lifecycle.test.ts b/tests/v2/lifecycle.test.ts new file mode 100644 index 0000000..f188b9b --- /dev/null +++ b/tests/v2/lifecycle.test.ts @@ -0,0 +1,118 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { transitionRemediation, assertExerciseTransition, persistentGap } from '../../server/v2/lifecycle.js'; +import { actionFixture, verificationFixture, scoringFixture, NOW } from './fixtures.js'; + +function requestFixture() { + return { + action: actionFixture(), next_state: 'ready_for_verification', + expected_record_version: 1, now: NOW, actor_uid: 'owner', evidence: scoringFixture().evidence, + }; +} + +test('completion becomes ready for verification, not verified or closed', () => { + const result = transitionRemediation(requestFixture()); + assert.equal(result.action.state, 'ready_for_verification'); + assert.equal(result.action.record_version, 2); + assert.equal(result.transition.verification_id, null); + assert.equal('score' in result, false); + assert.equal('readiness' in result, false); + for (const next_state of ['verified', 'closed']) { + assert.throws(() => transitionRemediation({ ...requestFixture(), next_state }), /INVALID_REMEDIATION_TRANSITION/); + } +}); +test('completion needs accepted evidence, assignment, and matching record version', () => { + assert.throws(() => transitionRemediation({ ...requestFixture(), evidence: [] }), /COMPLETION_EVIDENCE_REQUIRED/); + assert.throws(() => transitionRemediation({ + ...requestFixture(), action: { ...actionFixture(), owner_uid: null }, + }), /ACTION_OWNER_REQUIRED/); + assert.throws(() => transitionRemediation({ ...requestFixture(), expected_record_version: 0 }), /RECORD_VERSION_CONFLICT/); +}); +test('completion rejects cross-tenant evidence', () => { + const request = requestFixture(); + request.evidence[0]!.organization_id = 'org-b'; + assert.throws(() => transitionRemediation(request), /TENANT_MISMATCH/); +}); +test('verified transition requires independent scoped verification record', () => { + const request = { + ...requestFixture(), action: { ...actionFixture(), state: 'ready_for_verification' }, + next_state: 'verified', actor_uid: 'evaluator', verification: verificationFixture(), + }; + assert.equal(transitionRemediation(request).action.state, 'verified'); + for (const patch of [ + { verifier_uid: 'owner' }, { verifier_uid: 'other' }, { action_id: 'other' }, + { finding_ids: ['other-finding'] }, { result: 'inconclusive' }, { result: 'fail' }, + { retest_exercise_id: 'other-exercise' }, { completed_at: '2026-09-24T00:00:00.000Z' }, + ]) { + assert.throws(() => transitionRemediation({ ...request, verification: { ...verificationFixture(), ...patch } })); + } +}); +test('failed verification returns to work, never directly to closed', () => { + const result = transitionRemediation({ + ...requestFixture(), action: { ...actionFixture(), state: 'ready_for_verification' }, + next_state: 'failed_verification', actor_uid: 'evaluator', + verification: { ...verificationFixture(), result: 'fail' }, + }); + assert.equal(result.action.state, 'failed_verification'); + assert.throws(() => transitionRemediation({ + ...requestFixture(), action: { ...actionFixture(), state: 'failed_verification' }, next_state: 'closed', + }), /INVALID_REMEDIATION_TRANSITION/); +}); +test('closure still requires retained verification, not status alone', () => { + const request = { + ...requestFixture(), action: { ...actionFixture(), state: 'verified' }, next_state: 'closed', actor_uid: 'evaluator', + }; + assert.throws(() => transitionRemediation(request), /VALID_INDEPENDENT_VERIFICATION_REQUIRED/); + assert.equal(transitionRemediation({ ...request, verification: verificationFixture() }).action.state, 'closed'); +}); +test('all findings on an action must be covered by its verification', () => { + assert.throws(() => transitionRemediation({ + ...requestFixture(), action: { ...actionFixture(), finding_ids: ['finding-1', 'finding-2'], state: 'ready_for_verification' }, + next_state: 'verified', actor_uid: 'evaluator', verification: verificationFixture(), + }), /VALID_INDEPENDENT_VERIFICATION_REQUIRED/); +}); +test('exercise transitions cannot skip review or reopen historical completion', () => { + for (const [from, to] of [['draft', 'scheduled'], ['scheduled', 'ready'], ['ready', 'running'], ['running', 'paused'], ['paused', 'running'], ['running', 'review'], ['review', 'completed']]) { + assert.doesNotThrow(() => assertExerciseTransition(from, to)); + } + assert.throws(() => assertExerciseTransition('running', 'completed')); + assert.throws(() => assertExerciseTransition('completed', 'running')); + assert.throws(() => assertExerciseTransition('archived', 'draft')); +}); + +function occurrence(exercise_id: string, outcome: string, day: number) { + return { + organization_id: 'org-a', exercise_id, capability_id: 'evidence_preservation', + obligation_key: 'preserve-before-reimage', scope_key: 'production', + context_version_id: 'context-v1', scoring_policy_version: 'readiness-v1', + finalized_at: `2026-09-${String(day).padStart(2, '0')}T00:00:00.000Z`, outcome, + }; +} +test('persistent gap counts distinct comparable failures; unknown does not erase failures', () => { + const result = persistentGap([occurrence('ex-1', 'fail', 1), occurrence('ex-2', 'unknown', 2), occurrence('ex-3', 'fail', 3)]); + assert.equal(result.persistent, true); + assert.deepEqual(result.exercise_ids, ['ex-1', 'ex-3']); +}); +test('qualifying pass clears current streak, not historical records', () => { + const records = [occurrence('ex-1', 'fail', 1), occurrence('ex-2', 'fail', 2), occurrence('ex-3', 'pass', 3)]; + assert.equal(persistentGap(records).persistent, false); + assert.equal(records.length, 3); +}); +test('single failure and no data do not manufacture persistence', () => { + assert.equal(persistentGap([occurrence('ex-1', 'fail', 1)]).persistent, false); + assert.equal(persistentGap([]).persistent, false); +}); +test('duplicate exercise recalculations cannot inflate persistent failures', () => { + assert.throws(() => persistentGap([occurrence('ex-1', 'fail', 1), occurrence('ex-1', 'fail', 2)]), /DUPLICATE_EXERCISE_OCCURRENCE/); +}); +test('cross-tenant, context, scope or obligation history cannot be compared silently', () => { + const first = occurrence('ex-1', 'fail', 1); + for (const patch of [{ organization_id: 'org-b' }, { scope_key: 'other' }, { context_version_id: 'new' }, { obligation_key: 'easier-action' }]) { + assert.throws(() => persistentGap([first, { ...occurrence('ex-2', 'pass', 2), ...patch }]), /NON_COMPARABLE_OCCURRENCES/); + } +}); +test('conflicting simultaneous outcomes require explicit ordering, not favorable ID sorting', () => { + assert.throws(() => persistentGap([ + occurrence('ex-1', 'fail', 1), occurrence('ex-2', 'pass', 1), + ]), /AMBIGUOUS_OCCURRENCE_ORDER/); +}); diff --git a/tests/v2/scoring.test.ts b/tests/v2/scoring.test.ts new file mode 100644 index 0000000..5cb5d45 --- /dev/null +++ b/tests/v2/scoring.test.ts @@ -0,0 +1,225 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { calculateExerciseScore, canonicalJson } from '../../server/v2/scoring.js'; +import { scoringFixture } from './fixtures.js'; + +test('score is derived from retained weighted inputs, not an opaque percentage', () => { + const score = calculateExerciseScore(scoringFixture()); + assert.equal(score.result.score, 60); + assert.equal(score.result.earned_weight, 6); + assert.equal(score.result.planned_weight, 10); + assert.equal(score.result.coverage_percent, 100); + assert.equal(score.inputs.length, 4); + assert.deepEqual(score.inputs.map(input => input.points), [1, 0, 0.5, 1]); + assert.equal(score.capabilities[0]?.score, 60); + assert.match(score.input_sha256, /^[a-f0-9]{64}$/); +}); + +test('replaying frozen inputs is deterministic and does not mutate them', () => { + const fixture = scoringFixture(); + const before = structuredClone(fixture); + assert.deepEqual(calculateExerciseScore(fixture), calculateExerciseScore(fixture)); + assert.deepEqual(fixture, before); +}); + +test('record ordering does not change input digest or result', () => { + const fixture = scoringFixture(); + const expected = calculateExerciseScore(fixture); + fixture.criteria.reverse(); + fixture.observations.reverse(); + fixture.evidence.reverse(); + fixture.expected_actions.reverse(); + assert.deepEqual(calculateExerciseScore(fixture), expected); +}); + +test('missing observations withhold readiness, but show provisional measured subset', () => { + const fixture = scoringFixture(); + fixture.observations = fixture.observations.slice(0, 1); + const score = calculateExerciseScore(fixture); + assert.equal(score.result.score, null); + assert.equal(score.result.provisional_score, 100); + assert.equal(score.result.coverage_percent, 40); + assert.equal(score.result.status, 'insufficient_evidence'); +}); + +test('no observations is not tested, not zero and not a perfect score', () => { + const fixture = scoringFixture(); + fixture.observations = []; + const score = calculateExerciseScore(fixture); + assert.equal(score.result.status, 'not_tested'); + assert.equal(score.result.score, null); + assert.equal(score.result.provisional_score, null); + assert.equal(score.result.coverage_percent, 0); +}); + +test('all observed failures can produce a valid zero', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.measurement = { kind: 'completion', completed: false, omission_observed: true }; + fixture.observations[2]!.measurement = { kind: 'quality', rating: 'fail' }; + fixture.observations[3]!.measurement = { kind: 'deadline', elapsed_ms: 900_001 }; + assert.equal(calculateExerciseScore(fixture).result.score, 0); +}); + +test('missing observation coverage cannot be manufactured by dropping a criterion', () => { + const fixture = scoringFixture(); + fixture.criteria.splice(1, 1); + fixture.observations.splice(1, 1); + assert.throws(() => calculateExerciseScore(fixture), /SCENARIO_MANIFEST_MISMATCH/); +}); + +test('expected action manifest cannot be truncated', () => { + const fixture = scoringFixture(); + fixture.expected_actions.pop(); + assert.throws(() => calculateExerciseScore(fixture), /SCENARIO_MANIFEST_MISMATCH/); +}); + +test('omission requires observation of omission, not merely no completion', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.measurement = { kind: 'completion', completed: false, omission_observed: false }; + const result = calculateExerciseScore(fixture); + assert.equal(result.inputs[0]?.reason, 'omission_not_observed'); + assert.equal(result.result.score, null); +}); + +for (const source_kind of ['ai_advisory', 'simulation_artifact', 'legacy_import'] as const) { + test(`${source_kind} is not evidence of participant performance`, () => { + const fixture = scoringFixture(); + fixture.evidence[0]!.source_kind = source_kind; + const result = calculateExerciseScore(fixture); + assert.equal(result.result.score, null); + assert.equal(result.inputs[0]?.reason, 'ineligible_evidence'); + }); +} + +for (const status of ['pending', 'quarantined', 'unavailable'] as const) { + test(`${status} evidence cannot count toward readiness`, () => { + const fixture = scoringFixture(); + fixture.evidence[0]!.status = status; + assert.equal(calculateExerciseScore(fixture).result.score, null); + }); +} + +for (const collection of ['evidence', 'observations', 'criteria', 'expected_actions'] as const) { + test(`reject cross-tenant ${collection}`, () => { + const fixture = scoringFixture(); + fixture[collection][0]!.organization_id = 'org-b'; + assert.throws(() => calculateExerciseScore(fixture), /TENANT_MISMATCH/); + }); +} + +test('reject cross-tenant scenario', () => { + const fixture = scoringFixture(); + fixture.scenario.organization_id = 'org-b'; + assert.throws(() => calculateExerciseScore(fixture), /TENANT_MISMATCH/); +}); + +test('reject missing evidence references instead of silently dropping them', () => { + const fixture = scoringFixture(); + fixture.evidence = []; + assert.throws(() => calculateExerciseScore(fixture), /DANGLING_EVIDENCE_REFERENCE/); +}); + +test('reject evidence from a different exercise inside the same tenant', () => { + const fixture = scoringFixture(); + fixture.evidence[0]!.exercise_id = 'exercise-elsewhere'; + assert.throws(() => calculateExerciseScore(fixture), /EVIDENCE_EXERCISE_MISMATCH/); +}); + +test('reject duplicate accepted observations instead of selecting a favorable score', () => { + const fixture = scoringFixture(); + fixture.observations.push({ ...fixture.observations[0]!, id: 'other-revision' }); + assert.throws(() => calculateExerciseScore(fixture), /AMBIGUOUS_OBSERVATION_REVISION/); +}); + +test('draft observation is excluded; accepted observation requires human attribution', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.status = 'draft'; + assert.equal(calculateExerciseScore(fixture).inputs[0]?.reason, 'observation_not_accepted'); + fixture.observations[0]!.status = 'accepted'; + fixture.observations[0]!.accepted_by_uid = null; + assert.throws(() => calculateExerciseScore(fixture), /requires reviewer/); +}); + +test('reject superseded observation', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.status = 'superseded'; + assert.throws(() => calculateExerciseScore(fixture), /SUPERSEDED_OBSERVATION/); +}); + +test('reject mismatched measurement and out-of-range weights', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.measurement = { kind: 'sequence', followed: true }; + assert.throws(() => calculateExerciseScore(fixture), /MEASUREMENT_KIND_MISMATCH/); + fixture.criteria[0]!.weight = -1; + assert.throws(() => calculateExerciseScore(fixture)); +}); + +test('policy version is explicit and unknown future policies are rejected', () => { + const fixture = scoringFixture(); + fixture.exercise.scoring_policy_version = 'ai-chosen-policy'; + assert.throws(() => calculateExerciseScore(fixture), /UNSUPPORTED_SCORING_POLICY/); +}); + +test('running exercises and draft scenarios cannot become finalized scores', () => { + const fixture = scoringFixture(); + fixture.exercise.state = 'running'; + assert.throws(() => calculateExerciseScore(fixture), /EXERCISE_NOT_REVIEWABLE/); + fixture.exercise.state = 'review'; + fixture.scenario.state = 'draft'; + assert.throws(() => calculateExerciseScore(fixture), /SCENARIO_NOT_PUBLISHED/); +}); + +test('unknown observation stays unknown even with accepted evidence', () => { + const fixture = scoringFixture(); + fixture.observations[0]!.measurement = { kind: 'unknown', reason: 'Capture unavailable' }; + assert.equal(calculateExerciseScore(fixture).inputs[0]?.reason, 'unknown_measurement'); +}); +test('an exercised capability with only unknown observations is insufficient, not untested', () => { + const fixture = scoringFixture(); + fixture.observations.forEach(observation => { + observation.measurement = { kind: 'unknown', reason: 'Capture unavailable' }; + }); + const result = calculateExerciseScore(fixture); + assert.equal(result.result.status, 'insufficient_evidence'); + assert.equal(result.result.score, null); + assert.equal(result.result.provisional_score, null); +}); +test('an optional exploratory criterion cannot raise or dilute required score', () => { + const fixture = scoringFixture(); + fixture.criteria[1]!.required = false; + const result = calculateExerciseScore(fixture); + assert.equal(result.result.score, 86); + assert.equal(result.result.planned_weight, 7); + assert.equal(result.inputs[1]?.included, false); + assert.equal(result.inputs[1]?.reason, 'optional_not_in_required_score'); +}); +test('a declared capability cannot disappear from coverage because it has no criteria', () => { + const fixture = scoringFixture(); + fixture.scenario.capability_ids.push('recovery'); + assert.throws(() => calculateExerciseScore(fixture), /CAPABILITY_HAS_NO_REQUIRED_CRITERIA/); +}); + +test('changed measurement generates a new input digest without rewriting old result', () => { + const fixture = scoringFixture(); + const old = calculateExerciseScore(fixture); + fixture.observations[1]!.measurement = { kind: 'sequence', followed: true }; + const corrected = calculateExerciseScore(fixture); + assert.equal(old.result.score, 60); + assert.equal(corrected.result.score, 90); + assert.notEqual(old.input_sha256, corrected.input_sha256); +}); + +test('negative response time, duplicate evidence IDs and arbitrary score fields are rejected', () => { + const fixture = scoringFixture(); + fixture.observations[3]!.measurement = { kind: 'deadline', elapsed_ms: -1 }; + assert.throws(() => calculateExerciseScore(fixture)); + const second = scoringFixture(); + second.observations[0]!.evidence_ids = ['evidence-1', 'evidence-1']; + assert.throws(() => calculateExerciseScore(second), /Duplicate reference/); + assert.throws(() => calculateExerciseScore({ ...scoringFixture(), score: 100 })); +}); + +test('canonical serialization rejects undefined and sorts object keys', () => { + assert.equal(canonicalJson({ b: 2, a: 1 }), canonicalJson({ a: 1, b: 2 })); + assert.throws(() => canonicalJson(undefined)); +}); diff --git a/tsconfig.v2.json b/tsconfig.v2.json new file mode 100644 index 0000000..9e0f715 --- /dev/null +++ b/tsconfig.v2.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "rootDir": ".", + "outDir": ".v2-build", + "types": ["node"] + }, + "include": ["shared/v2/**/*.ts", "server/v2/**/*.ts", "tests/v2/**/*.ts"] +}