From 1460007981848eeb9e43085e0c26a40611e33f2a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 02:31:40 +0000 Subject: [PATCH] S25 wiring: policy-inheritance in shadow mode (SHIP_POLICY_SHADOW) with disagreement report Record-only comparison of resolvePolicy against mayDo, effectiveAuthority, resolveNetworkTier, normalizeEgressAllow, assertRepoAllowed and the enqueue daily budget. Default off installs nothing. Adds 'policy shadow-report'. Service accounts are id-only matches; their role is still undecided. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VqsBNqvaWezf1DwQrAnVgX --- src/cli.ts | 11 +- src/egress.ts | 14 +- src/governance.ts | 12 + src/ladder.ts | 6 +- src/policy-shadow.test.ts | 356 +++++++++++++++++++++++ src/policy-shadow.ts | 579 ++++++++++++++++++++++++++++++++++++++ src/repo-policy.ts | 25 ++ src/runtime.ts | 14 +- src/shadow-hook.ts | 91 ++++++ 9 files changed, 1103 insertions(+), 5 deletions(-) create mode 100644 src/policy-shadow.test.ts create mode 100644 src/policy-shadow.ts create mode 100644 src/shadow-hook.ts diff --git a/src/cli.ts b/src/cli.ts index 97ea2d0..5f19c63 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -42,6 +42,7 @@ import { attachEvidence, envAsked } from "./fix-evidence.js"; import type { Evidence } from "./verification.js"; import type { RepoRef } from "./git.js"; import { assertRepoAllowed, credentialFor, policyFromEnv } from "./repo-policy.js"; +import { readShadowSummary, renderShadowReport, shadowFile } from "./policy-shadow.js"; import type { RepoPolicyConfig } from "./repo-policy.js"; import { loadRepoContext, runNote } from "./repo-memory.js"; import { runAgent } from "./agent.js"; @@ -1597,7 +1598,15 @@ async function policyCommand(rest: string[]): Promise { " teploy-ship policy authority --roles admin,editor [--users a,b]\n" + " teploy-ship policy window set [--source ] --days mon-fri --start 09:00 --end 18:00 --tz \n" + " teploy-ship policy window remove|check [--source ]\n" + - " teploy-ship policy reviewers set [--users a,b] [--teams t]"; + " teploy-ship policy reviewers set [--users a,b] [--teams t]\n" + + " teploy-ship policy shadow-report [--json] where the S25 policy shadow (SHIP_POLICY_SHADOW=on) disagreed"; + + if (sub === "shadow-report") { + const file = shadowFile(); + const summary = await readShadowSummary(file); + process.stdout.write(args.flags.json === true ? `${JSON.stringify({ file, ...summary }, null, 2)}\n` : `${renderShadowReport(summary, file)}\n`); + return; + } if (sub === "show") { const runtime = await makeRuntime(args, config); diff --git a/src/egress.ts b/src/egress.ts index ad773bc..a8a3d22 100644 --- a/src/egress.ts +++ b/src/egress.ts @@ -29,6 +29,8 @@ */ /** What Ship records and reasons about. */ +import { shadowObserve } from "./shadow-hook.js"; + export type NetworkTier = "none" | "allowlist" | "open"; export const NETWORK_TIERS: readonly NetworkTier[] = ["none", "allowlist", "open"]; @@ -95,7 +97,10 @@ export function resolveNetworkTier( config?: string | undefined, ): NetworkTier | null { const chosen = parseNetworkTier(flag ?? env ?? config); - return chosen === null ? null : (chosen ?? DEFAULT_NETWORK_TIER); + const result = chosen === null ? null : (chosen ?? DEFAULT_NETWORK_TIER); + // S25 shadow (SHIP_POLICY_SHADOW): report the decision, never alter it. + shadowObserve({ point: "network", result }); + return result; } /** @@ -151,6 +156,13 @@ export function egressEntryError(entry: string): string | null { * host nobody can explain, on a run that has already been paid for. */ export function normalizeEgressAllow(entries: readonly string[] | undefined): string[] | undefined { + const result = normalizeEgressAllowEntries(entries); + // S25 shadow (SHIP_POLICY_SHADOW): report the decision, never alter it. A throw above is not reported. + shadowObserve({ point: "egress", result }); + return result; +} + +function normalizeEgressAllowEntries(entries: readonly string[] | undefined): string[] | undefined { if (entries === undefined) return undefined; const out: string[] = []; for (const raw of entries) { diff --git a/src/governance.ts b/src/governance.ts index e1c59a8..f774817 100644 --- a/src/governance.ts +++ b/src/governance.ts @@ -7,6 +7,7 @@ import { upsertByKey } from "./upsert.js"; import { stateDir } from "./run-store.js"; import { repoSlug } from "./observe.js"; import { normalizeRole } from "./users.js"; +import { shadowObserve } from "./shadow-hook.js"; import type { Role } from "./users.js"; /** @@ -101,6 +102,17 @@ export function mayDo( governance: Pick, action: AuthorityAction, principal: { user: string; role: string } | null | undefined, +): boolean { + const allowed = mayDoDecision(governance, action, principal); + // S25 shadow (SHIP_POLICY_SHADOW): report the decision, never alter it. + shadowObserve({ point: "mayDo", governance, action, principal, allowed }); + return allowed; +} + +function mayDoDecision( + governance: Pick, + action: AuthorityAction, + principal: { user: string; role: string } | null | undefined, ): boolean { if (principal === null || principal === undefined) return false; const grant = governance.authority[action]; diff --git a/src/ladder.ts b/src/ladder.ts index 2057122..4e74565 100644 --- a/src/ladder.ts +++ b/src/ladder.ts @@ -23,6 +23,7 @@ import type { ChangeClass } from "./change-class.js"; import type { TestOutcome } from "./tests.js"; import type { WorkflowEvent } from "@neutron-build/workflow"; import { verificationFactsFromEvents } from "./verification-summary.js"; +import { shadowObserve } from "./shadow-hook.js"; /** Contract 1's `verification` block, camel-cased for the project record. */ export interface ProjectVerification { @@ -102,7 +103,10 @@ export function effectiveAuthority(project: { }): Authority { const asked = project.authority ?? "send"; const capped = minAuthority(asked, authorityCap(project.verification)); - return project.neverAuto === true ? minAuthority(capped, "send") : capped; + const result = project.neverAuto === true ? minAuthority(capped, "send") : capped; + // S25 shadow (SHIP_POLICY_SHADOW): report the decision, never alter it. + shadowObserve({ point: "authority", project, result }); + return result; } export type RungName = "baseline" | "build" | "tests" | "preview" | "visual" | "flow" | "observe"; diff --git a/src/policy-shadow.test.ts b/src/policy-shadow.test.ts new file mode 100644 index 0000000..70e2ff6 --- /dev/null +++ b/src/policy-shadow.test.ts @@ -0,0 +1,356 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, test } from "node:test"; + +import { normalizeEgressAllow, resolveNetworkTier } from "./egress.js"; +import { DEFAULT_AUTHORITY, mayDo } from "./governance.js"; +import type { Governance } from "./governance.js"; +import { effectiveAuthority } from "./ladder.js"; +import { resolvePolicy } from "./policy-inheritance.js"; +import type { EffectivePolicy, PolicyLayer, PolicyLimits } from "./policy-inheritance.js"; +import { + flushShadow, + installPolicyShadow, + installPolicyShadowFromEnv, + parseShadowRecords, + readDeclaredLayers, + readShadowSummary, + renderShadowReport, + summariseShadow, +} from "./policy-shadow.js"; +import type { ShadowRecord, ShadowSink } from "./policy-shadow.js"; +import { assertRepoAllowed, RepoNotAllowedError } from "./repo-policy.js"; +import { assertDailyBudget, DailyBudgetExceededError } from "./runtime.js"; +import type { ShipRuntime } from "./runtime.js"; +import { setShadowObserver } from "./shadow-hook.js"; +import type { SpendStore } from "./spend.js"; + +const NOW = new Date("2026-10-04T12:00:00Z"); +const governance: Governance = { authority: DEFAULT_AUTHORITY, windows: {}, reviewers: [] }; + +function memorySink(): { sink: ShadowSink; records: ShadowRecord[] } { + const records: ShadowRecord[] = []; + return { records, sink: { append: async (r) => void records.push(r) } }; +} + +const layer = (kind: PolicyLayer["kind"], id: string, limits: PolicyLimits): PolicyLayer => ({ kind, id, limits }); +const declare = (...layers: PolicyLayer[]) => async () => ({ layers }); + +function install(layers: PolicyLayer[] = [], resolve?: (l: readonly PolicyLayer[], n: Date) => EffectivePolicy) { + const m = memorySink(); + installPolicyShadow({ sink: m.sink, now: () => NOW, declaredLayers: declare(...layers), env: {}, ...(resolve !== undefined ? { resolve } : {}) }); + return m; +} + +afterEach(() => { + installPolicyShadow(null); + setShadowObserver(null); +}); + +// ── flag off: nothing changes ───────────────────────────────────────────── + +test("flag off installs no observer: decisions are identical and nothing is written", async () => { + const dir = await mkdtemp(join(tmpdir(), "shadow-off-")); + const file = join(dir, "policy-shadow.jsonl"); + const env = { SHIP_POLICY_SHADOW_FILE: file } as NodeJS.ProcessEnv; + assert.equal(installPolicyShadowFromEnv(env), false); + assert.equal(installPolicyShadowFromEnv({ ...env, SHIP_POLICY_SHADOW: "off" }), false); + + const run = () => + JSON.stringify({ + a: mayDo(governance, "approve", { user: "e", role: "editor" }), + b: mayDo(governance, "auto", { user: "e", role: "editor" }), + c: mayDo(governance, "auto", null), + d: effectiveAuthority({ authority: "auto_normal", neverAuto: false }), + e: resolveNetworkTier("open", undefined, undefined), + f: normalizeEgressAllow([" GitHub.com ", ".npmjs.org"]), + g: assertRepoAllowed("https://github.com/o/r", { trust: "operator", config: {} }).cloneUrl, + }); + const off = run(); + await flushShadow(); + await assert.rejects(stat(file), "no record file is created while the flag is off"); + + // The same calls with the shadow ON return byte-identical results. + install(); + assert.equal(run(), off); + await flushShadow(); +}); + +test("flag on: only the exact value 'on' enables it", () => { + const env = { SHIP_POLICY_SHADOW_FILE: join(tmpdir(), "never-written.jsonl") } as NodeJS.ProcessEnv; + for (const v of ["", "1", "true", "ON-ish", "no"]) assert.equal(installPolicyShadowFromEnv({ ...env, SHIP_POLICY_SHADOW: v }), false, v); + assert.equal(installPolicyShadowFromEnv({ ...env, SHIP_POLICY_SHADOW: "on" }), true); +}); + +// ── real decision paths ─────────────────────────────────────────────────── + +test("mayDo: a declared organisation grant narrower than governance is a disagreement, and the outcome does not change", async () => { + const m = install([layer("org", "acme", { actions: { approve: { roles: ["admin"], users: [] } } })]); + const allowed = mayDo(governance, "approve", { user: "ed", role: "editor" }); + await flushShadow(); + assert.equal(allowed, true, "shadow never changes the outcome"); + assert.equal(m.records.length, 1); + const r = m.records[0]!; + assert.equal(r.type, "disagreement"); + assert.equal(r.kind, "existing-allows-policy-denies"); + assert.equal(r.point, "mayDo"); + assert.equal(r.dimension, "actions.approve"); + assert.equal(r.subject, "ed"); + assert.ok(r.layers.includes("org:acme"), "names the layer that refused"); +}); + +test("mayDo: with no declared layers the derived layers mirror governance, so nothing is recorded", async () => { + const m = install(); + for (const action of ["approve", "auto", "steer", "policies"] as const) { + for (const role of ["admin", "editor", "viewer", "wizard"]) mayDo(governance, action, { user: `u-${role}`, role }); + } + mayDo(governance, "approve", null); + await flushShadow(); + assert.deepEqual(m.records, []); +}); + +test("mayDo: a revoked user layer is a disagreement (existing allows, policy denies)", async () => { + const m = install([{ kind: "user", id: "ed", revokedAt: "2026-10-01T00:00:00Z", limits: {} }]); + assert.equal(mayDo(governance, "steer", { user: "ed", role: "editor" }), true); + await flushShadow(); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.kind, "existing-allows-policy-denies"); + assert.match(m.records[0]!.rule ?? "", /revoked/); +}); + +test("service account: id-only match, role is not consulted, and the record is tagged", async () => { + const bot = layer("service_account", "ci-bot", {}); + const byRole = install([bot]); + assert.equal(mayDo(governance, "approve", { user: "ci-bot", role: "admin" }), true, "existing code matches by role"); + await flushShadow(); + assert.equal(byRole.records.length, 1); + assert.equal(byRole.records[0]!.serviceAccount, true); + assert.equal(byRole.records[0]!.kind, "existing-allows-policy-denies"); + + // Named by id in the grant: both agree, nothing recorded. + const named: Governance = { ...governance, authority: { ...DEFAULT_AUTHORITY, approve: { roles: [], users: ["ci-bot"] } } }; + const byId = install([bot]); + assert.equal(mayDo(named, "approve", { user: "ci-bot", role: "viewer" }), true); + await flushShadow(); + assert.deepEqual(byId.records, []); +}); + +test("effectiveAuthority: a declared project layer that is stricter than the record is recorded; the result is unchanged", async () => { + const project = { repo: "o/r", authority: "auto_normal" as const, verification: { tests: "t", preview: { app: "a", smoke: "/" }, visual: true as const, observeWindowMin: 5 } }; + const quiet = install(); + assert.equal(effectiveAuthority(project), "auto_normal"); + await flushShadow(); + assert.deepEqual(quiet.records, [], "derived layer mirrors the record"); + + const m = install([layer("project", "o/r", { authority: "send" })]); + assert.equal(effectiveAuthority(project), "auto_normal"); + await flushShadow(); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.dimension, "authority"); + assert.equal(m.records[0]!.existing, "auto_normal"); + assert.equal(m.records[0]!.policy, "send"); + assert.equal(m.records[0]!.kind, "existing-allows-policy-denies"); +}); + +test("resolveNetworkTier: a tier above a declared organisation ceiling is recorded; the chosen tier is unchanged", async () => { + const m = install([layer("org", "acme", { network: "allowlist" })]); + assert.equal(resolveNetworkTier("open", undefined, undefined), "open"); + assert.equal(resolveNetworkTier("none", undefined, undefined), "none", "a tighter choice is not a disagreement"); + await flushShadow(); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.dimension, "network"); + assert.equal(m.records[0]!.existing, "open"); + assert.equal(m.records[0]!.policy, "allowlist"); +}); + +test("normalizeEgressAllow: entries outside a declared organisation allowlist are recorded; the list is unchanged", async () => { + const m = install([layer("org", "acme", { egressAllow: ["github.com", ".npmjs.org"], network: "allowlist" })]); + const out = normalizeEgressAllow(["github.com", "registry.npmjs.org", "evil.example"]); + await flushShadow(); + assert.deepEqual(out, ["github.com", "registry.npmjs.org", "evil.example"]); + assert.deepEqual(m.records.map((r) => r.existing), ["allow evil.example"]); + assert.equal(m.records[0]!.kind, "existing-allows-policy-denies"); +}); + +test("assertRepoAllowed: a refusal the policy would have allowed is recorded, and it still throws", async () => { + const m = install(); + assert.throws(() => assertRepoAllowed("https://github.com/o/r", { trust: "external", config: {} }), RepoNotAllowedError); + await flushShadow(); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.kind, "policy-allows-existing-denies"); + assert.equal(m.records[0]!.point, "repo"); + assert.match(m.records[0]!.note ?? "", /proxy/); +}); + +test("assertRepoAllowed: an allowed clone whose host a declared network ceiling closes is recorded", async () => { + const m = install([layer("org", "acme", { network: "none" })]); + const ref = assertRepoAllowed("https://github.com/o/r", { trust: "operator", config: {} }); + await flushShadow(); + assert.equal(ref.owner, "o"); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.kind, "existing-allows-policy-denies"); +}); + +function spendStore(initial = 0): SpendStore { + const holds = new Map(); + return { + async add() {}, + async get() { + return initial + [...holds.values()].reduce((a, b) => a + b, 0); + }, + async list() { + return []; + }, + async reserve(id: string, _s: string, _d: string, amount: number) { + holds.set(id, amount); + }, + async release(id: string) { + holds.delete(id); + }, + } as unknown as SpendStore; +} + +const budgetRuntime = (spend: SpendStore, project: unknown = null): Pick => + ({ spend, policies: { list: async () => [{ source: "web", policy: "propose", dailyBudgetUSD: 5 }] }, projects: { forRepo: async () => project } }) as never; + +test("assertDailyBudget: a weekly cap that today's spend plus this run exceeds is recorded; the run is still admitted", async () => { + const m = install(); + const project = { repo: "o/r", weeklyBudgetUSD: 1 }; + await assertDailyBudget(budgetRuntime(spendStore(1), project), { runId: "r1", source: "web", repo: "o/r", now: NOW }); + await flushShadow(); + const weekly = m.records.filter((r) => r.dimension === "weeklyBudgetUSD"); + assert.equal(weekly.length, 1); + assert.equal(weekly[0]!.kind, "existing-allows-policy-denies"); + assert.match(weekly[0]!.note ?? "", /lower-bounded/); + assert.equal(m.records.filter((r) => r.dimension === "dailyBudgetUSD").length, 0, "daily cap mirrors the existing one"); +}); + +test("assertDailyBudget: the daily refusal still throws, and an agreeing policy records nothing", async () => { + const m = install(); + await assert.rejects( + assertDailyBudget(budgetRuntime(spendStore(10)), { runId: "r2", source: "web", now: NOW }), + DailyBudgetExceededError, + ); + await flushShadow(); + assert.deepEqual(m.records, []); +}); + +test("assertDailyBudget: a declared organisation budget below the repo's own cap is a daily disagreement", async () => { + const m = install([layer("org", "acme", { dailyBudgetUSD: 3 })]); + const project = { repo: "o/r", dailyBudgetUSD: 20 }; + await assertDailyBudget(budgetRuntime(spendStore(4), project), { runId: "r3", source: "web", repo: "o/r", now: NOW }); + await flushShadow(); + const daily = m.records.filter((r) => r.dimension === "dailyBudgetUSD"); + assert.equal(daily.length, 1); + assert.equal(daily[0]!.kind, "existing-allows-policy-denies"); +}); + +// ── the shadow is isolated ──────────────────────────────────────────────── + +test("a broken sink and a malformed layers file never fail or alter a decision", async () => { + installPolicyShadow({ + sink: { append: async () => Promise.reject(new Error("disk full")) }, + now: () => NOW, + declaredLayers: async () => ({ layers: [], problem: "garbage" }), + env: {}, + }); + assert.equal(mayDo(governance, "approve", { user: "e", role: "editor" }), true); + assert.equal(resolveNetworkTier("open", undefined, undefined), "open"); + await flushShadow(); +}); + +test("a malformed declared-layers file is recorded as an error, not as agreement", async () => { + const dir = await mkdtemp(join(tmpdir(), "shadow-layers-")); + const path = join(dir, "policy-layers.json"); + await writeFile(path, "{not json"); + const m = memorySink(); + installPolicyShadow({ sink: m.sink, now: () => NOW, declaredLayers: () => readDeclaredLayers(path), env: {} }); + mayDo(governance, "approve", { user: "e", role: "editor" }); + await flushShadow(); + assert.equal(m.records.length, 1); + assert.equal(m.records[0]!.type, "error"); + assert.match(m.records[0]!.note ?? "", /declared layers ignored/); + assert.equal(summariseShadow(m.records).errors, 1); + assert.equal((await readDeclaredLayers(join(dir, "absent.json"))).problem, undefined, "an absent file is simply no layers"); +}); + +// ── negative control: a union-merge resolver must be visible ────────────── + +/** The bug S25 exists to prevent: combine layers by union / max instead of intersection / min. */ +function unionResolve(layers: readonly PolicyLayer[], now: Date): EffectivePolicy { + const p = resolvePolicy(layers, now); + const auths = layers.map((l) => l.limits.authority).filter((a): a is NonNullable => a !== undefined); + const order = ["propose", "send", "auto_trivial", "auto_normal"] as const; + const loosest = auths.sort((a, b) => order.indexOf(b) - order.indexOf(a))[0]; + const budgets = layers.map((l) => l.limits.dailyBudgetUSD).filter((b) => b !== undefined); + const daily = budgets.includes("unlimited") ? "unlimited" : Math.max(...(budgets as number[])); + return { ...p, authority: loosest ?? p.authority, dailyBudgetUSD: daily }; +} + +test("negative control: the correct resolver agrees on these paths, a union-merge resolver produces visible disagreements", async () => { + const project = { repo: "o/r", authority: "send" as const }; + const drive = async () => { + effectiveAuthority(project); + await assert.rejects( + assertDailyBudget(budgetRuntime(spendStore(6), { repo: "o/r", dailyBudgetUSD: 5 }), { runId: "nc", source: "web", repo: "o/r", now: NOW }), + DailyBudgetExceededError, + ); + await flushShadow(); + }; + + const correct = install(); + await drive(); + assert.deepEqual(correct.records, [], "intersection merge agrees with the existing decisions"); + + const buggy = install([], unionResolve); + await drive(); + const dims = buggy.records.map((r) => `${r.dimension}:${r.kind}`).sort(); + assert.deepEqual(dims, ["authority:policy-allows-existing-denies", "dailyBudgetUSD:policy-allows-existing-denies"]); +}); + +// ── report ──────────────────────────────────────────────────────────────── + +test("report: groups repeats, separates errors, tags service accounts, and states what it cannot know", async () => { + const m = install([layer("org", "acme", { actions: { approve: { roles: ["admin"], users: [] } } }), layer("service_account", "bot", {})]); + for (let i = 0; i < 3; i++) mayDo(governance, "approve", { user: "ed", role: "editor" }); + mayDo(governance, "approve", { user: "bot", role: "admin" }); + await flushShadow(); + const text = [...m.records.map((r) => JSON.stringify(r)), "not json", JSON.stringify({ v: 2 })].join("\n"); + const parsed = parseShadowRecords(text); + assert.equal(parsed.malformed, 2); + const s = summariseShadow(parsed.records, parsed.malformed); + assert.equal(s.disagreements, 4); + assert.equal(s.groups.find((g) => g.subject === "ed")?.count, 3); + assert.equal(s.serviceAccountDisagreements, 1); + const out = renderShadowReport(s, "x.jsonl"); + assert.match(out, /3x existing-allows-policy-denies/); + assert.match(out, /\[service-account\]/); + assert.match(out, /service-account role is NOT decided/i); + assert.match(out, /cannot give a disagreement rate/); + assert.match(out, /2 unreadable line/); + assert.equal(summariseShadow([]).disagreements, 0); +}); + +test("report: a missing record file reads as unknown, not as zero disagreements", async () => { + const dir = await mkdtemp(join(tmpdir(), "shadow-missing-")); + const s = await readShadowSummary(join(dir, "absent.jsonl")); + assert.equal(s.fileFound, false); + assert.match(renderShadowReport(s, "absent.jsonl"), /unknown, not zero/); +}); + +test("the file sink writes JSON lines the report command reads back", async () => { + const dir = await mkdtemp(join(tmpdir(), "shadow-file-")); + const file = join(dir, "nested", "policy-shadow.jsonl"); + const layers = join(dir, "layers.json"); + await writeFile(layers, JSON.stringify({ layers: [layer("org", "acme", { actions: { approve: { roles: ["admin"], users: [] } } })] })); + const env = { SHIP_POLICY_SHADOW: "on", SHIP_POLICY_SHADOW_FILE: file, SHIP_POLICY_LAYERS_FILE: layers } as NodeJS.ProcessEnv; + assert.equal(installPolicyShadowFromEnv(env), true); + assert.equal(mayDo(governance, "approve", { user: "ed", role: "editor" }), true); + await flushShadow(); + const { records, malformed } = parseShadowRecords(await readFile(file, "utf8")); + assert.equal(malformed, 0); + assert.equal(records.length, 1); + assert.equal(records[0]!.point, "mayDo"); +}); diff --git a/src/policy-shadow.ts b/src/policy-shadow.ts new file mode 100644 index 0000000..eb1d069 --- /dev/null +++ b/src/policy-shadow.ts @@ -0,0 +1,579 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { NETWORK_TIERS } from "./egress.js"; +import type { NetworkTier } from "./egress.js"; +import { AUTHORITIES } from "./ladder.js"; +import type { Authority } from "./ladder.js"; +import { AUTHORITY_ACTIONS } from "./governance.js"; +import type { Governance } from "./governance.js"; +import { explainRefusal, resolvePolicy, validateLayer } from "./policy-inheritance.js"; +import type { EffectivePolicy, PolicyLayer, PolicyLimits, PolicyRequest } from "./policy-inheritance.js"; +import { stateDir } from "./run-store.js"; +import { setShadowObserver, suppressShadow } from "./shadow-hook.js"; +import type { ShadowEvent, ShadowObserver, ShadowPrincipal, ShadowProjectLike } from "./shadow-hook.js"; + +/** + * S25 in SHADOW mode: run `resolvePolicy` next to every gated decision, record + * where it would have decided differently, and change nothing. + * + * Flag `SHIP_POLICY_SHADOW=on` (default off). Off installs no observer, so the + * decision points behave and print exactly as before. On, each decision point + * (shadow-hook.ts) reports its outcome AFTER deciding; this module builds the + * layers, resolves the policy, and appends one JSON line per DISAGREEMENT to + * `policy-shadow.jsonl` in the state directory (`SHIP_POLICY_SHADOW_FILE` + * overrides). `teploy-ship policy shadow-report` summarises that file. + * + * WHAT THE LAYERS ARE. Derived from data Ship already has, additive only: + * - org: a fully OPEN ceiling (any model/tool, network open, no spend limit, + * top authority) plus the governance grants as the organisation's action + * rules. Ship has no mandatory organisation limit today, so inventing one + * would manufacture disagreements. The one exception is the explicit + * `SHIP_DAILY_BUDGET_USD`, which is an operator-declared deployment cap. + * - project: the project record (authority + ladder + neverAuto, network, + * egress allowlist, weekly budget) and the daily cap the existing code + * ACTUALLY applied (repo > source > default). A source-policy budget with no + * project record is loaded as a project-rank layer `project:source:`. + * - user / service_account: the principal, with no limits of its own (silence + * in a human's layer means inherit). + * - declared (optional): `policy-layers.json` in the state directory + * (`SHIP_POLICY_LAYERS_FILE`). This is where an operator states the policy + * they WANT (an organisation budget, a revoked user, a network ceiling) so + * the shadow shows what adopting it would change. Declared layers are added + * to the derived ones; two layers of one level resolve to the stricter. + * + * Because derived layers mirror today's configuration, a deployment with no + * declared layers records few disagreements by construction. The shadow earns + * its keep once layers are declared, and on the dimensions Ship does not + * enforce at all today (weekly budget, repo host versus egress). + * + * SERVICE ACCOUNTS. The role of a service account is NOT decided (product + * confirmation owed, AUDIT_OPEN). The shadow therefore treats a service account + * as an id-only match, as policy-inheritance does: a role grant never admits + * it. Ship has no service-account principals yet; one appears only when a + * caller sets `serviceAccount` or a declared `service_account:` layer + * names the principal, and every such record is tagged. + * + * WHAT IS NOT RECORDED. Agreements are not counted (volume), so the report + * cannot give a disagreement RATE; it says so. A comparison the shadow could + * not make (bad layers file, resolution failure) is recorded as an `error` + * record, so "no disagreements" is never confused with "could not check". + */ + +export type DisagreementKind = "existing-allows-policy-denies" | "policy-allows-existing-denies"; + +export type ShadowPoint = ShadowEvent["point"]; + +export interface ShadowRecord { + v: 1; + at: string; + type: "disagreement" | "error"; + point: ShadowPoint; + dimension: string; + kind?: DisagreementKind; + /** Who or what the decision was about (principal, repo, source). */ + subject: string; + existing: string; + policy: string; + /** Layers that jointly set the policy limit. */ + layers: string[]; + rule?: string; + /** True when the subject is a service account: matched by id only, role never consulted. */ + serviceAccount?: boolean; + note?: string; +} + +export interface ShadowSink { + append(record: ShadowRecord): Promise; +} + +export const SHADOW_FLAG = "SHIP_POLICY_SHADOW"; + +export function shadowFlagOn(env: NodeJS.ProcessEnv = process.env): boolean { + return env[SHADOW_FLAG]?.trim().toLowerCase() === "on"; +} + +export const shadowFile = (env: NodeJS.ProcessEnv = process.env): string => + env.SHIP_POLICY_SHADOW_FILE ?? join(stateDir(), "policy-shadow.jsonl"); +export const layersFile = (env: NodeJS.ProcessEnv = process.env): string => + env.SHIP_POLICY_LAYERS_FILE ?? join(stateDir(), "policy-layers.json"); + +// ── layer loader ────────────────────────────────────────────────────────── + +export interface LayerInput { + governance?: Pick; + principal?: ShadowPrincipal | null; + project?: ShadowProjectLike | null; + /** Stand-in id when the daily cap came from a source policy and no project record exists. */ + sourceName?: string; + /** The daily cap the existing code applied (repo > source > default). */ + dailyCapUSD?: number; + declared?: readonly PolicyLayer[]; + env?: NodeJS.ProcessEnv; +} + +const UNATTRIBUTED = "(shadow-unattributed)"; + +function deploymentBudget(env: NodeJS.ProcessEnv): number | "unlimited" { + const raw = env.SHIP_DAILY_BUDGET_USD; + const n = raw === undefined || raw.trim() === "" ? NaN : Number(raw); + return Number.isFinite(n) && n > 0 ? n : "unlimited"; +} + +export function loadShadowLayers(input: LayerInput): { layers: PolicyLayer[]; problems: string[] } { + const env = input.env ?? process.env; + const problems: string[] = []; + const org: PolicyLimits = { + models: "*", + tools: "*", + network: "open", + egressAllow: "*", + dailyBudgetUSD: deploymentBudget(env), + weeklyBudgetUSD: "unlimited", + authority: "auto_normal", + provisioning: "apply", + workspace: "write", + }; + if (input.governance !== undefined) { + org.actions = Object.fromEntries(AUTHORITY_ACTIONS.map((a) => [a, input.governance!.authority[a]])) as PolicyLimits["actions"]; + } + const layers: PolicyLayer[] = [{ kind: "org", id: "deployment", limits: org }]; + + const projectId = input.project?.repo ?? (input.sourceName !== undefined ? `source:${input.sourceName}` : undefined); + if (projectId !== undefined) { + const p = input.project ?? {}; + const limits: PolicyLimits = { + // Mirrors effectiveAuthority's own default: a record that declares nothing is `send`. + authority: p.authority ?? "send", + ...(p.neverAuto === true ? { neverAuto: true } : {}), + ...(p.verification !== undefined ? { verification: p.verification } : {}), + ...(p.sandboxNetwork !== undefined ? { network: p.sandboxNetwork } : {}), + ...(p.sandboxEgressAllow !== undefined ? { egressAllow: p.sandboxEgressAllow } : {}), + ...(input.dailyCapUSD !== undefined && input.dailyCapUSD > 0 ? { dailyBudgetUSD: input.dailyCapUSD } : {}), + ...(p.weeklyBudgetUSD !== undefined ? { weeklyBudgetUSD: p.weeklyBudgetUSD } : {}), + }; + layers.push({ kind: "project", id: projectId, limits }); + } + + const principal = input.principal ?? null; + const principalId = principal?.user ?? UNATTRIBUTED; + const declaredForPrincipal = (input.declared ?? []).find( + (l) => (l.kind === "user" || l.kind === "service_account") && l.id === principalId, + ); + const isSA = principal?.serviceAccount === true || declaredForPrincipal?.kind === "service_account"; + + for (const l of input.declared ?? []) { + const bad = validateLayer(l); + if (bad.length > 0) { + problems.push(`declared layer ${l.kind}:${l.id}: ${bad.join("; ")}`); + continue; + } + if (l.kind === "org") layers.push(l); + else if (l.kind === "project") { + if (l.id === projectId) layers.push(l); + } else if (l.id === principalId) layers.push(l); + } + if (declaredForPrincipal === undefined) { + layers.push({ kind: isSA ? "service_account" : "user", id: principalId, limits: {} }); + } + return { layers, problems }; +} + +/** Parse the optional declared-layers file. Missing file is not a problem; a malformed one is reported. */ +export async function readDeclaredLayers(path: string): Promise<{ layers: PolicyLayer[]; problem?: string }> { + let text: string; + try { + text = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return { layers: [] }; + return { layers: [], problem: `cannot read ${path}: ${error instanceof Error ? error.message : String(error)}` }; + } + try { + const raw = JSON.parse(text) as unknown; + const arr = Array.isArray(raw) ? raw : (raw as { layers?: unknown })?.layers; + if (!Array.isArray(arr)) return { layers: [], problem: `${path}: expected an array of layers or {"layers": [...]}` }; + const ok: PolicyLayer[] = []; + for (const l of arr) { + const o = l as Partial; + if (!["org", "project", "user", "service_account"].includes(String(o?.kind)) || typeof o?.id !== "string" || typeof o?.limits !== "object" || o.limits === null) { + return { layers: [], problem: `${path}: every layer needs kind, id and limits` }; + } + ok.push(o as PolicyLayer); + } + return { layers: ok }; + } catch (error) { + return { layers: [], problem: `${path}: ${error instanceof Error ? error.message : String(error)}` }; + } +} + +// ── comparison ──────────────────────────────────────────────────────────── + +const tierIndex = (t: NetworkTier): number => NETWORK_TIERS.indexOf(t); + +interface Compared { + dimension: string; + existing: string; + policy: string; + layers: string[]; + rule?: string; + kind: DisagreementKind; + note?: string; +} + +function byRequest(policy: EffectivePolicy, request: PolicyRequest, existingAllowed: boolean, dimension: string, existing: string): Compared | null { + const refusal = explainRefusal(policy, request); + const policyAllows = refusal === null; + if (policyAllows === existingAllowed) return null; + return { + dimension, + existing, + policy: policyAllows ? "allow" : "deny", + layers: refusal?.layers ?? [], + ...(refusal !== null ? { rule: refusal.rule } : {}), + kind: existingAllowed ? "existing-allows-policy-denies" : "policy-allows-existing-denies", + }; +} + +export interface ShadowObserverOptions { + sink: ShadowSink; + env?: NodeJS.ProcessEnv; + now?: () => Date; + /** Declared layers; defaults to reading `layersFile(env)` on every decision. */ + declaredLayers?: () => Promise<{ layers: PolicyLayer[]; problem?: string }>; + /** Test seam: the resolver under observation. Defaults to `resolvePolicy`; a buggy one proves the shadow notices. */ + resolve?: (layers: readonly PolicyLayer[], now: Date) => EffectivePolicy; +} + +const inflight = new Set>(); + +/** Resolves when every comparison started so far has been recorded (tests; the CLI exits without it, which is fine for a log). */ +export async function flushShadow(): Promise { + while (inflight.size > 0) await Promise.allSettled([...inflight]); +} + +/** Builds the observer. Exported so tests install it with a fake sink and fake layers. */ +export function createShadowObserver(options: ShadowObserverOptions): ShadowObserver { + const env = options.env ?? process.env; + const now = options.now ?? (() => new Date()); + const readDeclared = options.declaredLayers ?? (() => readDeclaredLayers(layersFile(env))); + const resolve = options.resolve ?? resolvePolicy; + + const emit = async (point: ShadowPoint, subject: string, serviceAccount: boolean, c: Compared | null): Promise => { + if (c === null) return; + await options.sink.append({ + v: 1, + at: now().toISOString(), + type: "disagreement", + point, + dimension: c.dimension, + kind: c.kind, + subject, + existing: c.existing, + policy: c.policy, + layers: c.layers, + ...(c.rule !== undefined ? { rule: c.rule } : {}), + ...(serviceAccount ? { serviceAccount: true } : {}), + ...(c.note !== undefined ? { note: c.note } : {}), + }); + }; + const fail = (point: ShadowPoint, subject: string, message: string): Promise => + options.sink.append({ v: 1, at: now().toISOString(), type: "error", point, dimension: "-", subject, existing: "-", policy: "-", layers: [], note: message }); + + const run: ShadowObserver = async (event) => { + // The observer is suppressed (shadow-hook) while it runs; resolvePolicy calls effectiveAuthority. + const declared = await readDeclared(); + const principalOf = (e: ShadowEvent): ShadowPrincipal | null => (e.point === "mayDo" ? (e.principal ?? null) : null); + let subject = ""; + let sa = false; + try { + let project: ShadowProjectLike | null = null; + let dailyCap: number | undefined; + let sourceName: string | undefined; + if (event.point === "authority") { + project = event.project; + sourceName = "(unnamed-project)"; // used only when the record carries no repo + } + if (event.point === "budget") { + project = await event.loadProject().catch(() => null); + dailyCap = event.budget; + sourceName = event.source; + } + const principal = principalOf(event); + if (event.point === "mayDo" && principal === null) return; // no principal: existing denies, nothing to resolve for + subject = + event.point === "mayDo" ? (principal?.user ?? "") : + event.point === "repo" ? event.url : + event.point === "budget" ? `${event.source}${event.repo !== undefined ? ` ${event.repo}` : ""}` : + event.point === "authority" ? (event.project.repo ?? "(project)") : "(context-free)"; + + const { layers, problems } = loadShadowLayers({ + env, + ...(event.point === "mayDo" ? { governance: event.governance } : {}), + principal, + project, + ...(sourceName !== undefined ? { sourceName } : {}), + ...(dailyCap !== undefined ? { dailyCapUSD: dailyCap } : {}), + declared: declared.layers, + }); + if (declared.problem !== undefined) await fail(event.point, subject, `declared layers ignored: ${declared.problem}`); + for (const p of problems) await fail(event.point, subject, `declared layer ignored: ${p}`); + const policy = suppressShadow(() => resolve(layers, now())); + sa = policy.principal?.kind === "service_account"; + + const found: Array = []; + switch (event.point) { + case "mayDo": + found.push( + byRequest(policy, { kind: "action", action: event.action, principal: { user: principal!.user, role: principal!.role } }, event.allowed, `actions.${event.action}`, event.allowed ? "allow" : "deny"), + ); + break; + case "authority": { + const idx = (a: Authority | "none"): number => (a === "none" ? -1 : AUTHORITIES.indexOf(a)); + const have = idx(event.result); + const want = idx(policy.authority); + if (want !== have) { + found.push({ + dimension: "authority", + existing: event.result, + policy: policy.authority, + layers: policy.explanations.authority.decidedBy, + kind: want < have ? "existing-allows-policy-denies" : "policy-allows-existing-denies", + ...(policy.explanations.authority.note !== undefined ? { note: policy.explanations.authority.note } : {}), + }); + } + break; + } + case "network": + // The tier is a CHOICE (flag > env > config), not a ceiling: only a tier above the policy is a disagreement. + if (event.result !== null && tierIndex(event.result) > tierIndex(policy.network)) { + found.push({ + dimension: "network", + existing: event.result, + policy: policy.network, + layers: policy.explanations.network.decidedBy, + kind: "existing-allows-policy-denies", + note: "resolveNetworkTier has no project context: only organisation and declared org layers can apply", + }); + } + break; + case "egress": + for (const entry of event.result ?? []) { + found.push( + byRequest(policy, { kind: "egress", destination: entry }, true, "egressAllow", `allow ${entry}`), + ); + } + break; + case "repo": + // Proxy: a repository is cloned from INSIDE the sandbox, so its host must be reachable under the policy's egress rules. + if (event.host !== null) { + const c = byRequest(policy, { kind: "egress", destination: event.host }, event.allowed, "egressAllow", event.allowed ? `repo ${event.host} allowed` : `repo ${event.host} refused`); + found.push(c === null ? null : { ...c, note: "proxy: repo allowlist versus the policy's egress rules for the clone host" }); + } + break; + case "budget": { + const spent = event.committed === null ? 0 : Math.max(0, event.committed - event.estimate); + const common = { spentUSD: spent, costUSD: event.estimate }; + found.push(byRequest(policy, { kind: "spend", scope: "daily", ...common }, event.existingAllowed, "dailyBudgetUSD", event.existingAllowed ? "allow" : "deny")); + // Weekly spend is not enforced today and no weekly total is read: today's committed spend is a LOWER BOUND. + const w = byRequest(policy, { kind: "spend", scope: "weekly", ...common }, true, "weeklyBudgetUSD", "not enforced"); + found.push(w === null ? null : { ...w, note: "weekly spend lower-bounded by today's committed spend" }); + break; + } + } + for (const c of found) await emit(event.point, subject, sa, c); + } catch (error) { + await fail(event.point, subject, `shadow comparison failed: ${error instanceof Error ? error.message : String(error)}`); + } + }; + return (event) => { + const p = Promise.resolve(run(event)); + inflight.add(p); + void p.finally(() => inflight.delete(p)).catch(() => {}); + return p; + }; +} + +// ── sink and installation ───────────────────────────────────────────────── + +export function fileSink(path: string): ShadowSink { + return { + async append(record) { + await mkdir(dirname(path), { recursive: true }); + await appendFile(path, `${JSON.stringify(record)}\n`); + }, + }; +} + +let installed = false; + +/** Installs the file-backed shadow when `SHIP_POLICY_SHADOW=on`. Off: does nothing at all. Idempotent. */ +export function installPolicyShadowFromEnv(env: NodeJS.ProcessEnv = process.env): boolean { + if (!shadowFlagOn(env) || installed) return shadowFlagOn(env); + installed = true; + const inner = createShadowObserver({ sink: fileSink(shadowFile(env)), env }); + // A failure to WRITE the record must not surface anywhere; shadow-hook swallows it, this keeps one line for the operator. + setShadowObserver(async (event) => { + try { + await inner(event); + } catch (error) { + process.stderr.write(`[ship] policy shadow: could not record (${error instanceof Error ? error.message : String(error)})\n`); + } + }); + return true; +} + +/** Test seam: install an observer with an injected sink, or remove it. */ +export function installPolicyShadow(options: ShadowObserverOptions | null): void { + installed = options !== null; + setShadowObserver(options === null ? null : createShadowObserver(options)); +} + +// ── report ──────────────────────────────────────────────────────────────── + +export interface ShadowGroup { + point: ShadowPoint; + dimension: string; + kind: DisagreementKind; + subject: string; + count: number; + first: string; + last: string; + existing: string; + policy: string; + layers: string[]; + rule?: string; + serviceAccount: boolean; +} + +export interface ShadowSummary { + disagreements: number; + errors: number; + malformedLines: number; + byKind: Record; + byDimension: Record; + groups: ShadowGroup[]; + errorSamples: string[]; + serviceAccountDisagreements: number; + notes: string[]; + /** False when the record file does not exist: the shadow never ran (or flag off). Unknown, not zero. */ + fileFound: boolean; +} + +export const REPORT_NOTES: readonly string[] = [ + "Service-account role is NOT decided (product confirmation owed). Service accounts are treated as id-only matches: a role grant never admits one. Records tagged [service-account] reflect that assumption, not a settled rule.", + "Only disagreements and shadow errors are recorded. Agreeing decisions are not counted, so this report cannot give a disagreement rate; zero disagreements means none were seen, not that every decision was checked.", + "Layers are derived from existing governance, project, egress and ladder data and mirror today's configuration; add declared layers (policy-layers.json) to see what a real organisation policy would change.", + "Shadow only: nothing here changed an outcome. Decision points without project or principal context (network tier, egress allowlist) can only see organisation-level layers.", +]; + +export function parseShadowRecords(text: string): { records: ShadowRecord[]; malformed: number } { + const records: ShadowRecord[] = []; + let malformed = 0; + for (const line of text.split("\n")) { + if (line.trim() === "") continue; + try { + const r = JSON.parse(line) as ShadowRecord; + if (r?.v === 1 && (r.type === "disagreement" || r.type === "error") && typeof r.point === "string") records.push(r); + else malformed++; + } catch { + malformed++; + } + } + return { records, malformed }; +} + +export function summariseShadow(records: readonly ShadowRecord[], malformed = 0): ShadowSummary { + const byKind: Record = { "existing-allows-policy-denies": 0, "policy-allows-existing-denies": 0 }; + const byDimension: Record = {}; + const groups = new Map(); + const errorSamples: string[] = []; + let errors = 0; + let disagreements = 0; + let sa = 0; + for (const r of records) { + if (r.type === "error") { + errors++; + if (r.note !== undefined && !errorSamples.includes(r.note) && errorSamples.length < 5) errorSamples.push(r.note); + continue; + } + if (r.kind === undefined) continue; + disagreements++; + byKind[r.kind]++; + byDimension[r.dimension] = (byDimension[r.dimension] ?? 0) + 1; + if (r.serviceAccount === true) sa++; + const key = [r.point, r.dimension, r.kind, r.subject, r.existing, r.policy].join("\u0000"); + const g = groups.get(key); + if (g === undefined) { + groups.set(key, { + point: r.point, + dimension: r.dimension, + kind: r.kind, + subject: r.subject, + count: 1, + first: r.at, + last: r.at, + existing: r.existing, + policy: r.policy, + layers: r.layers, + ...(r.rule !== undefined ? { rule: r.rule } : {}), + serviceAccount: r.serviceAccount === true, + }); + } else { + g.count++; + if (r.at < g.first) g.first = r.at; + if (r.at > g.last) g.last = r.at; + } + } + return { + disagreements, + errors, + malformedLines: malformed, + byKind, + byDimension, + groups: [...groups.values()].sort((a, b) => b.count - a.count || a.point.localeCompare(b.point)), + errorSamples, + serviceAccountDisagreements: sa, + notes: [...REPORT_NOTES], + fileFound: true, + }; +} + +export function renderShadowReport(s: ShadowSummary, file: string): string { + const out: string[] = []; + out.push(`policy shadow report (${file})`); + if (!s.fileFound) out.push(`no record file: the shadow has not recorded anything here (${SHADOW_FLAG} is off, or never ran). This is unknown, not zero disagreements.`); + out.push( + `${s.disagreements} disagreement(s): ${s.byKind["existing-allows-policy-denies"]} existing allows / policy denies, ` + + `${s.byKind["policy-allows-existing-denies"]} policy allows / existing denies`, + ); + if (s.errors > 0) out.push(`${s.errors} shadow error(s): comparisons that could not be made (unknown, NOT agreement)`); + if (s.malformedLines > 0) out.push(`${s.malformedLines} unreadable line(s) skipped`); + if (s.serviceAccountDisagreements > 0) out.push(`${s.serviceAccountDisagreements} involve a service account (id-only match assumed)`); + const dims = Object.entries(s.byDimension).sort((a, b) => b[1] - a[1]); + if (dims.length > 0) out.push(`by dimension: ${dims.map(([d, n]) => `${d}=${n}`).join(", ")}`); + for (const g of s.groups) { + out.push(""); + out.push(`${g.count}x ${g.kind}${g.serviceAccount ? " [service-account]" : ""}`); + out.push(` ${g.point} / ${g.dimension} / ${g.subject}: existing ${g.existing}, policy ${g.policy}`); + if (g.layers.length > 0) out.push(` decided by: ${g.layers.join(", ")}`); + if (g.rule !== undefined) out.push(` rule: ${g.rule}`); + out.push(` seen ${g.first} .. ${g.last}`); + } + for (const e of s.errorSamples) out.push(`error: ${e}`); + out.push(""); + for (const n of s.notes) out.push(`note: ${n}`); + return out.join("\n"); +} + +export async function readShadowSummary(path: string): Promise { + let text = ""; + let fileFound = true; + try { + text = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + fileFound = false; + } + const { records, malformed } = parseShadowRecords(text); + return { ...summariseShadow(records, malformed), fileFound }; +} diff --git a/src/repo-policy.ts b/src/repo-policy.ts index cae2167..41f1ba3 100644 --- a/src/repo-policy.ts +++ b/src/repo-policy.ts @@ -1,5 +1,6 @@ import { parseRepoUrl } from "./git.js"; import type { RepoRef } from "./git.js"; +import { shadowObserve } from "./shadow-hook.js"; /** * Which repositories Ship will point a credential at, and which credential. @@ -185,6 +186,30 @@ export function effectiveAllowlist(config: RepoPolicyConfig): RepoAllowEntry[] { export function assertRepoAllowed( url: string, options: { trust: RepoTrust; config?: RepoPolicyConfig }, +): RepoRef { + // S25 shadow (SHIP_POLICY_SHADOW): report the decision either way, never alter it. + let ref: RepoRef; + try { + ref = assertRepoAllowedDecision(url, options); + } catch (error) { + shadowObserve({ point: "repo", url, trust: options.trust, host: repoHost(url), allowed: false, reason: error instanceof Error ? error.message : String(error) }); + throw error; + } + shadowObserve({ point: "repo", url, trust: options.trust, host: ref.base === "file://" ? null : repoHost(url), allowed: true }); + return ref; +} + +function repoHost(url: string): string | null { + try { + return parseRepoUrl(url).base.replace(/^[a-z+]+:\/\//i, "").toLowerCase() || null; + } catch { + return null; + } +} + +function assertRepoAllowedDecision( + url: string, + options: { trust: RepoTrust; config?: RepoPolicyConfig }, ): RepoRef { const config = options.config ?? policyFromEnv(); let ref: RepoRef; diff --git a/src/runtime.ts b/src/runtime.ts index 3c7aba0..ba6c3cb 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -77,6 +77,8 @@ import type { RecoveryTuning } from "./durable.js"; import type { ProposeInput as IntakeProposeInput, IntakeTask as IntakeTaskType } from "./intake.js"; import { FileEventStore, RunMetaStore } from "./run-store.js"; import { askEnv, readDeploymentAsks } from "./deployment-asks.js"; +import { installPolicyShadowFromEnv } from "./policy-shadow.js"; +import { shadowObserve } from "./shadow-hook.js"; import type { RunMeta } from "./run-store.js"; export type { RunMeta } from "./run-store.js"; @@ -436,6 +438,7 @@ export interface ShipRuntime { } export function fileRuntime(): ShipRuntime { + installPolicyShadowFromEnv(); // no-op unless SHIP_POLICY_SHADOW=on const store = new FileEventStore(); const meta = new RunMetaStore(); const projects = new FileProjectStore(); @@ -505,6 +508,7 @@ export async function nucleusRuntime( // `db` is a seam, not a feature: NucleusPgwire already takes a PoolLike so a // test can watch the statements it emits, and the reads this runtime issues // (listMeta above all) are worth asserting on rather than trusting. + installPolicyShadowFromEnv(); // no-op unless SHIP_POLICY_SHADOW=on const db = options?.db ?? new NucleusPgwire(url, owner); // Bring the shared schema to the shape this binary expects BEFORE handing // back a runtime. A rolling deploy runs old and new processes against one @@ -791,15 +795,21 @@ export async function assertDailyBudget( if ((await spend.held?.(options.runId)) === true) return; const budget = await dailyBudgetForSource(runtime, source, options.repo); - if (!(budget > 0)) return; // <= 0 disables the cap for that source, as in the worker + const estimate = estimatedRunCostUSD(); + const shadowProject = async () => (options.repo === undefined ? null : await runtime.projects.forRepo(options.repo)); + if (!(budget > 0)) { + // <= 0 disables the cap for that source, as in the worker + shadowObserve({ point: "budget", source, ...(options.repo !== undefined ? { repo: options.repo } : {}), budget, committed: null, estimate, existingAllowed: true, loadProject: shadowProject }); + return; + } const day = spendDay(options.now ?? new Date()); - const estimate = estimatedRunCostUSD(); // Reserve BEFORE reading the total, exactly as the sweep does: two surfaces // admitting at once must see each other's commitment rather than both reading // the same room. Over-reserving briefly is the safe direction. await spend.reserve(options.runId, source, day, estimate); const committed = await spend.get(source, day); + shadowObserve({ point: "budget", source, ...(options.repo !== undefined ? { repo: options.repo } : {}), budget, committed, estimate, existingAllowed: !(committed > budget), loadProject: shadowProject }); if (committed > budget) { await spend.release(options.runId).catch(() => {}); throw new DailyBudgetExceededError(source, budget, committed); diff --git a/src/shadow-hook.ts b/src/shadow-hook.ts new file mode 100644 index 0000000..b3bed05 --- /dev/null +++ b/src/shadow-hook.ts @@ -0,0 +1,91 @@ +import type { AuthorityAction, Governance } from "./governance.js"; +import type { Authority, ProjectVerification } from "./ladder.js"; +import type { NetworkTier } from "./egress.js"; + +/** + * The seam between Ship's existing decision points and the S25 policy shadow. + * + * WHY a leaf module. The decision points (governance.ts, ladder.ts, egress.ts, + * repo-policy.ts, runtime.ts) cannot import policy-inheritance.ts: that module + * imports governance, ladder and egress at load time, so the reverse edge is a + * cycle whose first victim is a top-level constant. This file imports only + * TYPES (erased at compile), so every decision point can call it freely. + * + * WHY an observer and not a return value. Shadow mode never changes an + * outcome. A decision point reports what it decided, AFTER deciding, to an + * observer that is null unless `SHIP_POLICY_SHADOW=on` installed one + * (policy-shadow.ts). Off, the cost is one null comparison and the decision is + * exactly what it was. The observer is isolated: a throw, or a rejected + * promise, is swallowed here so the shadow can never fail a request. + */ + +export interface ShadowPrincipal { + user: string; + role: string; + /** Set by callers that know the principal is an automation identity. Ship has none today. */ + serviceAccount?: boolean; +} + +export interface ShadowProjectLike { + repo?: string; + authority?: Authority; + neverAuto?: boolean; + verification?: ProjectVerification; + sandboxNetwork?: NetworkTier; + sandboxEgressAllow?: string[]; + dailyBudgetUSD?: number; + weeklyBudgetUSD?: number; +} + +export type ShadowEvent = + | { point: "mayDo"; governance: Pick; action: AuthorityAction; principal: ShadowPrincipal | null | undefined; allowed: boolean } + | { point: "authority"; project: ShadowProjectLike; result: Authority } + | { point: "network"; result: NetworkTier | null } + | { point: "egress"; result: string[] | undefined } + | { point: "repo"; url: string; trust: string; host: string | null; allowed: boolean; reason?: string } + | { + point: "budget"; + source: string; + repo?: string; + /** The daily cap the existing code applied. <= 0 means it applied none. */ + budget: number; + /** Spend committed today INCLUDING this run's own reservation. Null when no spend was read (cap disabled). */ + committed: number | null; + estimate: number; + existingAllowed: boolean; + loadProject: () => Promise; + }; + +export type ShadowObserver = (event: ShadowEvent) => void | Promise; + +let observer: ShadowObserver | null = null; +let suppressed = 0; + +export function setShadowObserver(next: ShadowObserver | null): void { + observer = next; +} + +export function shadowActive(): boolean { + return observer !== null && suppressed === 0; +} + +/** Run `fn` with the shadow muted: the observer itself calls decision points (resolvePolicy -> effectiveAuthority). */ +export function suppressShadow(fn: () => T): T { + suppressed++; + try { + return fn(); + } finally { + suppressed--; + } +} + +/** Report a decision that has ALREADY been made. Never throws, never returns anything, never delays the caller. */ +export function shadowObserve(event: ShadowEvent): void { + if (observer === null || suppressed > 0) return; + try { + const out = suppressShadow(() => observer!(event)); + if (out !== undefined && typeof (out as Promise).catch === "function") (out as Promise).catch(() => {}); + } catch { + // The shadow must never fail the decision it is watching. + } +}