From 23d0204b3508a544c02507a6228a97851a9082b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:17:38 +0000 Subject: [PATCH] Wire tool-manifest (S24) observe-only: tool validate dry run, conformance shadow, flagged event envelope headers SHIP_TOOL_MANIFEST=shadow|on records conformance findings from executeAction, never blocking. SHIP_EVENT_ENVELOPE=on adds two signed headers to run webhooks; off is byte-identical. http-client example gains verifyEvent/EventDedupe. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VqsBNqvaWezf1DwQrAnVgX --- docs/HTTP_CLIENT.md | 34 ++++ examples/http-client.mjs | 55 +++++ src/agent.ts | 14 ++ src/args.ts | 1 + src/cli.ts | 33 +++ src/notify.ts | 52 +++++ src/tool-manifest-shadow.ts | 189 +++++++++++++++++ src/tool-manifest-wiring.test.ts | 335 +++++++++++++++++++++++++++++++ 8 files changed, 713 insertions(+) create mode 100644 src/tool-manifest-shadow.ts create mode 100644 src/tool-manifest-wiring.test.ts diff --git a/docs/HTTP_CLIENT.md b/docs/HTTP_CLIENT.md index 2569b32..9fd20b8 100644 --- a/docs/HTTP_CLIENT.md +++ b/docs/HTTP_CLIENT.md @@ -56,3 +56,37 @@ SDK. Raw durable storage is deliberately absent. Workspace activity is the recorded view, not an event-stream subscription. Contract tests do not prove a particular live provider/forge configuration; installation and live lifecycle receipts are separate acceptance evidence. + +## Verifying webhook events + +Ship's run webhook (`SHIP_NOTIFY_URL`) is signed as before: +`X-Teploy-Signature: sha256=hex(HMAC-SHA256(secret, timestamp + "." + body))`. +That is unchanged and is all a receiver needs. Delivery is at-least-once, so a +receiver should also dedupe on `X-Teploy-Delivery`. + +If the operator sets `SHIP_EVENT_ENVELOPE=on` (default off) and a +`SHIP_NOTIFY_SECRET`, each delivery that carries an `event_seq` also gets two +additive headers; the body and every existing header stay byte-identical: + +- `X-Teploy-Event`: a JSON envelope (`eventId`, `schemaVersion`, `type`, + `cursor`, `occurredAt`, `data`). `eventId` equals `X-Teploy-Delivery`, `cursor` + is `event_seq`, and `data.payloadSha256` is the SHA-256 of the exact body. +- `X-Teploy-Event-Signature`: `sha256=hex(HMAC-SHA256(secret, X-Teploy-Timestamp + "." + ))`. + +`verifyEvent` and `EventDedupe` in `examples/http-client.mjs` check all of that +(signature, a 5 minute replay window, schema major, body binding) and report a +retried delivery as `duplicate: true`, which you acknowledge without acting on: + +```js +import { EventDedupe, verifyEvent } from './examples/http-client.mjs'; +const dedupe = new EventDedupe(); +// `raw` is the unparsed request body string; headers are lower-cased (node:http) +const r = verifyEvent(req.headers, raw, process.env.SHIP_NOTIFY_SECRET, { dedupe }); +if (!r.ok) return reply(401, r.reason); // never act on an unverified event +if (r.duplicate) return reply(200, 'duplicate'); // already handled +handle(JSON.parse(raw), r.event.cursor); +``` + +Verify against the raw bytes, not a re-serialised body. Keep the dedupe set +durable if your receiver restarts. Run webhooks carry no `requestId`: a run +notification is not the answer to one of your calls. diff --git a/examples/http-client.mjs b/examples/http-client.mjs index f66e679..4ee2435 100644 --- a/examples/http-client.mjs +++ b/examples/http-client.mjs @@ -1,3 +1,5 @@ +import { createHash, createHmac, timingSafeEqual } from 'node:crypto'; + /** A dependency-free Node 22 client for Ship's existing HTTP surfaces. */ export class ShipHTTPError extends Error { constructor(status, operation, detail) { @@ -79,3 +81,56 @@ export class ShipClient { }, 'follow-up'); } } + +// ---- Event verification (webhook receivers) ------------------------------- +// A port of verifyEvent/EventDedupe from src/tool-manifest.ts, kept dependency +// free. Only meaningful when the Ship operator set SHIP_EVENT_ENVELOPE=on and a +// notify secret: Ship then adds X-Teploy-Event and X-Teploy-Event-Signature to +// each run webhook (body and X-Teploy-Signature unchanged). + +/** Bounded set of seen event ids (FIFO eviction). */ +export class EventDedupe { + #seen = new Set(); + #capacity; + constructor(capacity = 10_000) { this.#capacity = capacity; } + has(id) { return this.#seen.has(id); } + add(id) { + this.#seen.add(id); + if (this.#seen.size > this.#capacity) this.#seen.delete(this.#seen.values().next().value); + } +} + +/** + * Verify one delivery. `headers` is a plain object with lower-cased names. + * Order: authenticate, freshness, parse, major version, body binding, dedupe. + * Only an event that passed everything is remembered, so a forged one cannot + * poison the dedupe set. A duplicate returns { ok: true, duplicate: true }: + * acknowledge it (2xx) and do nothing. Returns { ok: false, reason } otherwise. + */ +export function verifyEvent(headers, body, secret, { nowMs = Date.now(), replayWindowMs = 300_000, dedupe } = {}) { + const envelope = headers['x-teploy-event']; + const signature = headers['x-teploy-event-signature']; + const timestamp = headers['x-teploy-timestamp']; + if (typeof envelope !== 'string' || typeof signature !== 'string') return { ok: false, reason: 'no-envelope' }; + const sig = /^sha256=([0-9a-f]{64})$/.exec(signature); + if (!sig) return { ok: false, reason: 'malformed-signature' }; + if (!/^\d{1,12}$/.test(timestamp ?? '')) return { ok: false, reason: 'bad-timestamp' }; + if (!secret) return { ok: false, reason: 'bad-signature' }; + const expected = createHmac('sha256', secret).update(`${timestamp}.${envelope}`).digest(); + const given = Buffer.from(sig[1], 'hex'); + if (given.length !== expected.length || !timingSafeEqual(given, expected)) return { ok: false, reason: 'bad-signature' }; + if (Math.abs(nowMs - Number(timestamp) * 1000) > replayWindowMs) return { ok: false, reason: 'expired' }; + let event; + try { event = JSON.parse(envelope); } catch { return { ok: false, reason: 'malformed-body' }; } + if (!event || typeof event !== 'object' || typeof event.eventId !== 'string' || event.eventId === '' || + typeof event.type !== 'string' || typeof event.schemaVersion !== 'string' || !Number.isInteger(event.cursor) || + event.cursor < 0 || typeof event.occurredAt !== 'string') return { ok: false, reason: 'malformed-envelope' }; + const version = /^(\d+)\.(\d+)/.exec(event.schemaVersion); + if (!version) return { ok: false, reason: 'malformed-envelope' }; + if (Number(version[1]) !== 1) return { ok: false, reason: 'unsupported-major' }; + // The envelope travels in headers; this ties it to the body actually received. + if (event.data?.payloadSha256 !== createHash('sha256').update(body).digest('hex')) return { ok: false, reason: 'body-mismatch' }; + if (dedupe?.has(event.eventId)) return { ok: true, duplicate: true, eventId: event.eventId }; + dedupe?.add(event.eventId); + return { ok: true, duplicate: false, event }; +} diff --git a/src/agent.ts b/src/agent.ts index 512eb8d..695deff 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -21,6 +21,7 @@ import { formatObservation, systemPrompt } from "./prompt.js"; import { scrub } from "./redact.js"; import { RecoveryTracker, defaultRecoveryConfig } from "./recovery.js"; import type { RecoveryConfig } from "./recovery.js"; +import { observeToolCall } from "./tool-manifest-shadow.js"; export interface AgentStep { index: number; @@ -548,6 +549,19 @@ export async function executeAction( timeoutMs?: number, scriptSuffix?: string, useKernel = true, +): Promise { + const result = await runAction(executor, action, timeoutMs, scriptSuffix, useKernel); + // S24 shadow: report the call AFTER it ran; a no-op unless SHIP_TOOL_MANIFEST is set. + void observeToolCall(action); + return result; +} + +async function runAction( + executor: AgentExecutor, + action: Extract, + timeoutMs?: number, + scriptSuffix?: string, + useKernel = true, ): Promise { const opts = timeoutMs !== undefined ? { timeoutMs } : {}; const suffix = scriptSuffix ?? String(Date.now()); diff --git a/src/args.ts b/src/args.ts index 0963e62..4802491 100644 --- a/src/args.ts +++ b/src/args.ts @@ -46,6 +46,7 @@ export const COMMAND_FLAGS: Record = { enqueue: { boolean: ["json", "plan", "critic", "settle", "tests"], value: ["repo", "model", "store", "nucleus-url"] }, evidence: { boolean: ["json"], value: ["test-command", "test-timeout-ms", "observe-service"] }, policy: { boolean: ["json"], value: ["roles", "users", "teams", "source", "days", "start", "end", "tz", "store", "nucleus-url"] }, + tool: { boolean: ["json"], value: ["grant"] }, resume: { value: ["model", "sandbox", "sandbox-token", "sandbox-image", "sandbox-network", "max-steps"] }, approve: { boolean: ["handoff"], value: ["model", "sandbox", "sandbox-token", "sandbox-image", "sandbox-network"] }, deny: { boolean: ["handoff"], value: ["model", "sandbox", "sandbox-token", "sandbox-image", "sandbox-network"] }, diff --git a/src/cli.ts b/src/cli.ts index d6bf5aa..ede4f9c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -45,6 +45,7 @@ import type { Evidence } from "./verification.js"; import type { RepoRef } from "./git.js"; import { assertRepoAllowed, credentialFor, policyFromEnv } from "./repo-policy.js"; import { readShadowSummary, renderShadowReport, shadowFile } from "./policy-shadow.js"; +import { dryRunManifest, parseGrant, renderDryRun } from "./tool-manifest-shadow.js"; import { detectRepository } from "./stack-import.js"; import { renderProposal, stackDetectEnabled } from "./stack-propose.js"; import type { RepoPolicyConfig } from "./repo-policy.js"; @@ -136,6 +137,9 @@ Usage: teploy-ship policy window remove [--source ] (no --source = the global window) teploy-ship policy window check [--source ] is auto allowed right now? teploy-ship policy reviewers set [--users a,b] [--teams t] (both empty = remove) + teploy-ship tool validate [--grant ] [--json] + dry run: is a tool manifest well formed, and what would + the admin grant actually allow? Stores and enforces nothing teploy-ship audit export the run history (what ran, cost, PRs) [--format csv|json] [--since ] [--until ] teploy-ship runs list durable runs @@ -1630,6 +1634,33 @@ async function projectCommand(rest: string[]): Promise { * The buyer half of P2-3 (governance.ts): per-user authority, auto windows, * required reviewers. The dashboard's Policies page edits the same store. */ +/** + * S24: `tool validate` is the read-only install dry run. It reads a manifest + * and an admin-entered grant, prints validation and the effective permissions + * (intersection, with the excess reported), and writes nothing. Exit 1 when the + * manifest is invalid so a script can gate on it. + */ +async function toolCommand(rest: string[]): Promise { + const args = parseArgs(rest, COMMAND_FLAGS.tool); + const [sub, file] = args.positional; + if (sub !== "validate" || file === undefined) fail("usage: teploy-ship tool validate [--grant ] [--json]"); + let manifest: unknown; + try { + manifest = JSON.parse(readFileSync(file, "utf8")); + } catch (error) { + fail(`cannot read manifest ${file}: ${error instanceof Error ? error.message : String(error)}`); + } + let grant; + try { + grant = parseGrant(typeof args.flags.grant === "string" ? args.flags.grant : undefined); + } catch (error) { + fail(`bad --grant: ${error instanceof Error ? error.message : String(error)}`); + } + const report = dryRunManifest(manifest, grant); + process.stdout.write(args.flags.json === true ? `${JSON.stringify(report, null, 2)}\n` : `${renderDryRun(report)}\n`); + if (!report.validation.ok) process.exitCode = 1; +} + async function policyCommand(rest: string[]): Promise { const config = loadConfig(); const [sub, second, third] = rest; @@ -2406,6 +2437,8 @@ async function main(): Promise { return projectCommand(rest); case "policy": return policyCommand(rest); + case "tool": + return toolCommand(rest); case "audit": return auditCommand(rest); case "resume": diff --git a/src/notify.ts b/src/notify.ts index 07ab6c9..0f67269 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,6 +1,9 @@ import type { ScanFinding } from "./findings.js"; import type { ChangeClass } from "./change-class.js"; import { rungsForWire, type Rung } from "./ladder.js"; +import { canonicalJson, EVENT_SCHEMA_VERSION } from "./tool-manifest.js"; +import type { EventEnvelope } from "./tool-manifest.js"; +import { createHash, createHmac } from "node:crypto"; /** * A4 — outbound notifications: one short Slack message when a run needs @@ -351,9 +354,55 @@ export async function signWebhookBody( return { "X-Teploy-Timestamp": ts, "X-Teploy-Signature": `sha256=${mac}` }; } +/** + * S24: the signed EventEnvelope as an ADDITIVE pair of headers, behind + * `SHIP_EVENT_ENVELOPE=on` (default off). The body and the existing + * X-Teploy-Timestamp / X-Teploy-Signature are untouched, so a receiver that + * has never heard of envelopes is byte-compatible whether the flag is on or off. + * + * X-Teploy-Event: the envelope as JSON (eventId, cursor, type, ...) + * X-Teploy-Event-Signature: sha256=hex(HMAC(secret, + "." + that header value)) + * + * The envelope rides in headers, not the body, because the body can be 64 KiB + * and headers cannot; `data.payloadSha256` binds the envelope to the exact body + * that was sent, so a receiver verifies the envelope with verifyEvent, then + * compares the hash. eventId is the delivery id (stable across retries) so the + * receiver's dedupe matches X-Teploy-Delivery; cursor is event_seq. An event + * with no event_seq has no honest cursor, so it carries NO envelope. There is no + * requestId: a run notification is not the answer to a client call. + */ +export function eventEnvelopeEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return (env.SHIP_EVENT_ENVELOPE ?? "").trim().toLowerCase() === "on"; +} + +export function envelopeHeaders( + event: RunNotification, + deliveryId: string | undefined, + secret: string, + body: string, + timestamp: string, +): Record { + if (secret === "" || event.eventSeq === undefined || !Number.isInteger(event.eventSeq) || event.eventSeq < 0) return {}; + const eventId = deliveryId ?? `${event.runId}:${event.status}:${event.eventName ?? ""}:${event.eventSeq}`; + const envelope: EventEnvelope = { + eventId, + schemaVersion: EVENT_SCHEMA_VERSION, + type: `run.${event.status}`, + cursor: event.eventSeq, + occurredAt: event.eventAt ?? new Date(Number(timestamp) * 1000).toISOString(), + data: { runId: event.runId, status: event.status, payloadSha256: createHash("sha256").update(body).digest("hex") }, + }; + // Header values are ByteStrings: escape anything non-ASCII (the same JSON). + const value = canonicalJson(envelope).replace(/[^\x20-\x7e]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`); + const mac = createHmac("sha256", secret).update(`${timestamp}.${value}`).digest("hex"); + return { "X-Teploy-Event": value, "X-Teploy-Event-Signature": `sha256=${mac}` }; +} + export function webhookNotifier(options?: { webhookUrl?: string; secret?: string; + /** Add the signed-envelope headers (default: SHIP_EVENT_ENVELOPE=on). */ + eventEnvelope?: boolean; publicUrl?: string; log?: (line: string) => void; fetchImpl?: typeof fetch; @@ -363,6 +412,7 @@ export function webhookNotifier(options?: { const publicUrl = options?.publicUrl ?? process.env.SHIP_PUBLIC_URL ?? ""; const log = options?.log ?? ((line: string) => process.stderr.write(line + "\n")); const fetchImpl = options?.fetchImpl ?? fetch; + const envelopeOn = options?.eventEnvelope ?? eventEnvelopeEnabled(); if (webhookUrl === "") return { enabled: false, runEvent: async () => true }; return { enabled: true, @@ -371,6 +421,7 @@ export function webhookNotifier(options?: { const body = JSON.stringify(runWebhookPayload(event, publicUrl)); try { const headers = await signWebhookBody(secret, body); + const envelope = envelopeOn ? envelopeHeaders(event, deliveryId, secret, body, headers["X-Teploy-Timestamp"] ?? "") : {}; const response = await fetchImpl(webhookUrl, { method: "POST", headers: { @@ -379,6 +430,7 @@ export function webhookNotifier(options?: { // recognise a repeat instead of acting on it twice. ...(deliveryId !== undefined ? { "X-Teploy-Delivery": deliveryId } : {}), ...headers, + ...envelope, }, body, }); diff --git a/src/tool-manifest-shadow.ts b/src/tool-manifest-shadow.ts new file mode 100644 index 0000000..f07b11f --- /dev/null +++ b/src/tool-manifest-shadow.ts @@ -0,0 +1,189 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { stateDir } from "./run-store.js"; +import { conformanceChecks, effectivePermissions, hasExcess, validateManifest } from "./tool-manifest.js"; +import type { AdminGrant, ConformanceFinding, Effect, ObservedCall, ToolManifest, ValidationResult } from "./tool-manifest.js"; + +/** + * S24 wired OBSERVE-ONLY: the tool manifest contract next to Ship's real tool + * path, enforcing nothing. + * + * Flag `SHIP_TOOL_MANIFEST=shadow|on` (default off). Off, `observeToolCall` is + * one env read and a return: executeAction (agent.ts) behaves exactly as before. + * Both settings only RECORD: `shadow` appends one JSON line per non-conforming + * call to `tool-manifest-shadow.jsonl` in the state directory + * (`SHIP_TOOL_MANIFEST_SHADOW_FILE` overrides); `on` additionally writes a one + * line summary to stderr so an operator watching a run sees it. Neither ever + * blocks, quarantines or alters a call. Enforcement stays with the credential + * scope and the executor's own controls (egress allowlist, read-only mounts), + * which is what manifest rule 3 says a declaration is not. + * + * WHICH MANIFEST. Ship has no installed-tool registry, so there is nothing to + * look a manifest up in. The operator names one file, `SHIP_TOOL_MANIFEST_FILE`, + * that declares Ship's own loop actions as tools (`bash`, `python`, `edit`, + * `create`). No file means nothing is checked and nothing is recorded: an + * "unknown-tool" finding for every call would be noise, not information. A file + * that cannot be read or does not validate is recorded ONCE as an `error` + * record, so "no findings" is never confused with "could not check". + * + * WHAT IS OBSERVABLE. The loop sees which action ran: bash/python are `exec`, + * edit/create are `write`. It does not see the network hosts a command + * contacted or the secret values the executor injected, so `hosts` and + * `secretValues` are empty here and `undeclared-host`, `secret-in-argv` and + * `secret-in-prompt` cannot fire from this path. Closing that needs the + * executor/credential layer to report them; recorded as open in the report. + */ + +export type ToolManifestMode = "off" | "shadow" | "on"; + +export function toolManifestMode(env: NodeJS.ProcessEnv = process.env): ToolManifestMode { + const v = (env.SHIP_TOOL_MANIFEST ?? "").trim().toLowerCase(); + return v === "shadow" || v === "on" ? v : "off"; +} + +export function toolManifestShadowFile(env: NodeJS.ProcessEnv = process.env): string { + return env.SHIP_TOOL_MANIFEST_SHADOW_FILE ?? join(stateDir(), "tool-manifest-shadow.jsonl"); +} + +export type ShadowRecord = + | { at: string; kind: "finding"; tool: string; finding: ConformanceFinding } + | { at: string; kind: "error"; error: string }; + +type Loaded = { manifest: ToolManifest } | { error: string }; +const loaded = new Map(); +const reportedErrors = new Set(); + +function loadManifest(path: string): Loaded { + const cached = loaded.get(path); + if (cached !== undefined) return cached; + let result: Loaded; + try { + const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); + const v = validateManifest(parsed); + result = v.ok ? { manifest: parsed as ToolManifest } : { error: `manifest ${path} is invalid: ${v.errors.join("; ")}` }; + } catch (error) { + result = { error: `manifest ${path} unreadable: ${error instanceof Error ? error.message : String(error)}` }; + } + loaded.set(path, result); + return result; +} + +/** Test seam: forget cached manifests and reported errors. */ +export function resetToolManifestShadow(): void { + loaded.clear(); + reportedErrors.clear(); +} + +/** What the loop actually saw, as an ObservedCall. See the header for what is missing. */ +export function observedFromAction(action: { kind: string; code?: string }): ObservedCall { + const effects: Effect[] = action.kind === "bash" || action.kind === "python" ? ["exec"] : ["write"]; + return { tool: action.kind, effects, hosts: [], argv: typeof action.code === "string" ? [action.code] : [], secretValues: [] }; +} + +async function record(file: string, rec: ShadowRecord): Promise { + await mkdir(dirname(file), { recursive: true }); + await appendFile(file, `${JSON.stringify(rec)}\n`, "utf8"); +} + +/** + * Report a call that ALREADY happened. Never throws, never returns anything the + * caller uses, never delays it (the write is not awaited). Returns the pending + * writes only so a test can wait for them. + */ +export function observeToolCall(action: { kind: string; code?: string }, env: NodeJS.ProcessEnv = process.env): Promise { + const mode = toolManifestMode(env); + if (mode === "off") return Promise.resolve(); + try { + const path = env.SHIP_TOOL_MANIFEST_FILE; + if (path === undefined || path === "") return Promise.resolve(); + const file = toolManifestShadowFile(env); + const at = new Date().toISOString(); + const m = loadManifest(path); + const pending: Promise[] = []; + if ("error" in m) { + // loadManifest caches, so this branch repeats; record only the first. + if (!reportedErrors.has(path)) { + reportedErrors.add(path); + pending.push(record(file, { at, kind: "error", error: m.error })); + if (mode === "on") process.stderr.write(`[tool-manifest] ${m.error}\n`); + } + } else { + for (const finding of conformanceChecks(m.manifest, observedFromAction(action))) { + pending.push(record(file, { at, kind: "finding", tool: action.kind, finding })); + if (mode === "on") process.stderr.write(`[tool-manifest] ${finding.kind}: ${finding.detail}\n`); + } + } + return Promise.all(pending.map((p) => p.catch(() => {}))).then(() => {}); + } catch { + // The shadow must never fail the call it is watching. + return Promise.resolve(); + } +} + +export async function readShadowRecords(file: string): Promise { + let text: string; + try { + text = await readFile(file, "utf8"); + } catch { + return []; + } + return text.split("\n").filter((l) => l !== "").map((l) => JSON.parse(l) as ShadowRecord); +} + +// ------------------------------------------------------------ dry-run view + +interface PermissionLists { effects: string[]; hosts: string[]; scopes: string[] } + +export interface DryRunReport { + validation: ValidationResult; + /** Empty when the manifest did not validate: a malformed manifest gets nothing. */ + tools: { tool: string; granted: PermissionLists; excess: PermissionLists; hasExcess: boolean }[]; + grant: AdminGrant; + /** Always true: this view persists nothing and grants nothing. */ + dryRun: true; +} + +/** Parse an admin grant from `--grant`: inline JSON or a path to a JSON file. Absent grants nothing. */ +export function parseGrant(value: string | undefined): AdminGrant { + if (value === undefined) return { effects: [], hosts: [], scopes: [] }; + const text = value.trimStart().startsWith("{") ? value : readFileSync(value, "utf8"); + const g = JSON.parse(text) as Record; + for (const k of Object.keys(g)) if (!["effects", "hosts", "scopes"].includes(k)) throw new Error(`grant: unknown field "${k}"`); + const list = (k: string): string[] => { + const v = g[k]; + if (v === undefined) return []; + if (!Array.isArray(v) || v.some((x) => typeof x !== "string")) throw new Error(`grant.${k} must be an array of strings`); + return v as string[]; + }; + return { effects: list("effects") as Effect[], hosts: list("hosts"), scopes: list("scopes") }; +} + +export function dryRunManifest(manifest: unknown, grant: AdminGrant): DryRunReport { + const validation = validateManifest(manifest); + const tools = validation.ok + ? effectivePermissions(manifest as ToolManifest, grant).map((e) => ({ tool: e.tool, granted: e.granted, excess: e.excess, hasExcess: hasExcess(e) })) + : []; + return { validation, tools, grant, dryRun: true }; +} + +export function renderDryRun(report: DryRunReport): string { + const out: string[] = []; + const v = report.validation; + out.push(v.ok ? "manifest: valid (a valid manifest is a claim; it grants nothing)" : "manifest: INVALID"); + for (const e of v.errors) out.push(` error: ${e}`); + for (const w of v.warnings) out.push(` warning: ${w}`); + if (v.ok) { + const j = (a: string[]): string => (a.length === 0 ? "-" : a.join(",")); + for (const t of report.tools) { + out.push(`tool ${t.tool}`); + out.push(` granted: effects ${j(t.granted.effects)} hosts ${j(t.granted.hosts)} scopes ${j(t.granted.scopes)}`); + out.push(t.hasExcess + ? ` requested but NOT granted: effects ${j(t.excess.effects)} hosts ${j(t.excess.hosts)} scopes ${j(t.excess.scopes)}` + : " requested nothing beyond the grant"); + } + } + out.push("dry run: nothing was installed, stored or enforced"); + return out.join("\n"); +} diff --git a/src/tool-manifest-wiring.test.ts b/src/tool-manifest-wiring.test.ts new file mode 100644 index 0000000..464e009 --- /dev/null +++ b/src/tool-manifest-wiring.test.ts @@ -0,0 +1,335 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, createHmac } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterEach, mock, test } from "node:test"; + +import type { AgentExecutor, ExecResult } from "@neutron-build/agents"; + +import { executeAction } from "./agent.js"; +import { runWebhookPayload, webhookNotifier } from "./notify.js"; +import type { RunNotification } from "./notify.js"; +import { observeToolCall, readShadowRecords, resetToolManifestShadow } from "./tool-manifest-shadow.js"; +import { CursorTracker, EventDedupe, verifyEvent } from "./tool-manifest.js"; +import type { ToolManifest } from "./tool-manifest.js"; + +const run = promisify(execFile); +const CLI = join(dirname(fileURLToPath(import.meta.url)), "cli.js"); + +function tool(name: string, effects: string[]): Record { + return { + name, input: { type: "object" }, output: { type: "object" }, + permissions: { effects, hosts: [], scopes: [] }, + secrets: [], secretTransport: "none", timeoutMs: 30_000, cancellation: "kill", approval: "never", + }; +} +const manifest = (tools: Record[]): ToolManifest => + ({ schemaVersion: "1.0", name: "ship-loop", version: "1.0.0", tools }) as unknown as ToolManifest; + +// ------------------------------------------------------------ webhook envelope + +const EVENT: RunNotification = { runId: "run-1", status: "waiting", eventName: "turn-1-approval", eventSeq: 7, eventAt: "2026-10-04T10:00:00.000Z", task: "fix ünïcode" }; + +async function deliver(opts: { eventEnvelope?: boolean; secret?: string; event?: RunNotification }) { + const seen: { url: string; init: RequestInit }[] = []; + const fetchImpl = (async (url: string, init: RequestInit) => { + seen.push({ url, init }); + return { ok: true, status: 200 } as Response; + }) as unknown as typeof fetch; + mock.method(Date, "now", () => 1_790_000_000_000); + try { + const n = webhookNotifier({ webhookUrl: "http://sink/hook", secret: opts.secret ?? "k", fetchImpl, log: () => {}, ...(opts.eventEnvelope !== undefined ? { eventEnvelope: opts.eventEnvelope } : {}) }); + assert.equal(await n.runEvent(opts.event ?? EVENT, "run-1:waiting:turn-1-approval:7"), true); + } finally { + mock.restoreAll(); + } + assert.equal(seen.length, 1); + return { url: seen[0]!.url, init: seen[0]!.init, headers: seen[0]!.init.headers as Record, body: String(seen[0]!.init.body) }; +} + +afterEach(() => { + delete process.env.SHIP_EVENT_ENVELOPE; + delete process.env.SHIP_TOOL_MANIFEST; + delete process.env.SHIP_TOOL_MANIFEST_FILE; + delete process.env.SHIP_TOOL_MANIFEST_SHADOW_FILE; + resetToolManifestShadow(); +}); + +test("envelope flag off (default and explicit): the delivery is byte-identical to the pre-S24 shape", async () => { + // Golden built WITHOUT the new code path: the exact headers and body the notifier sent before. + const body = JSON.stringify(runWebhookPayload(EVENT, undefined)); + const ts = "1790000000"; + const golden = { + "content-type": "application/json", + "X-Teploy-Delivery": "run-1:waiting:turn-1-approval:7", + "X-Teploy-Timestamp": ts, + "X-Teploy-Signature": `sha256=${createHmac("sha256", "k").update(`${ts}.${body}`).digest("hex")}`, + }; + for (const eventEnvelope of [undefined, false]) { + const d = await deliver(eventEnvelope === undefined ? {} : { eventEnvelope }); + assert.equal(d.body, body); + assert.deepEqual(d.headers, golden); + assert.deepEqual(Object.keys(d.headers), Object.keys(golden), "same header order too"); + } + // an unrecognised env value is also off + process.env.SHIP_EVENT_ENVELOPE = "yes"; + assert.deepEqual((await deliver({})).headers, golden); +}); + +test("envelope flag on: body and existing headers unchanged, two extra headers verify with verifyEvent", async () => { + const off = await deliver({ eventEnvelope: false }); + const on = await deliver({ eventEnvelope: true }); + assert.equal(on.body, off.body, "body is byte-identical"); + const { "X-Teploy-Event": ev, "X-Teploy-Event-Signature": evSig, ...rest } = on.headers; + assert.deepEqual(rest, off.headers, "every pre-existing header is unchanged"); + assert.ok(ev !== undefined && evSig !== undefined); + assert.match(ev!, /^[\x20-\x7e]+$/, "header value is ASCII"); + + const dedupe = new EventDedupe(); + const cursors = new CursorTracker(6); + const opts = { nowMs: 1_790_000_000_000, dedupe, cursors }; + const first = verifyEvent(ev!, { timestamp: on.headers["X-Teploy-Timestamp"]!, signature: evSig! }, "k", opts); + assert.equal(first.ok, true); + assert.ok(first.ok && !first.duplicate); + if (first.ok && !first.duplicate) { + assert.equal(first.event.eventId, "run-1:waiting:turn-1-approval:7"); + assert.equal(first.event.cursor, 7); + assert.equal(first.event.type, "run.waiting"); + assert.equal((first.event.data as { payloadSha256: string }).payloadSha256, createHash("sha256").update(on.body).digest("hex"), "bound to the exact body"); + } + assert.equal(cursors.resumeFrom, 7); + // a retry of the same delivery is recognised as a duplicate + const again = verifyEvent(ev!, { timestamp: on.headers["X-Teploy-Timestamp"]!, signature: evSig! }, "k", opts); + assert.ok(again.ok && again.duplicate); +}); + +test("envelope: tamper, wrong secret and a body swap are all caught (negative controls)", async () => { + const on = await deliver({ eventEnvelope: true }); + const h = { timestamp: on.headers["X-Teploy-Timestamp"]!, signature: on.headers["X-Teploy-Event-Signature"]! }; + const ev = on.headers["X-Teploy-Event"]!; + const o = { nowMs: 1_790_000_000_000 }; + assert.deepEqual(verifyEvent(ev.replace('"cursor":7', '"cursor":8'), h, "k", o), { ok: false, reason: "bad-signature" }); + assert.deepEqual(verifyEvent(ev, h, "other", o), { ok: false, reason: "bad-signature" }); + // the BODY signature header must not verify the envelope (distinct MAC input) + assert.deepEqual(verifyEvent(ev, { ...h, signature: on.headers["X-Teploy-Signature"]! }, "k", o), { ok: false, reason: "bad-signature" }); + // a different body no longer matches the bound hash + const r = verifyEvent(ev, h, "k", o); + assert.ok(r.ok && !r.duplicate); + if (r.ok && !r.duplicate) assert.notEqual((r.event.data as { payloadSha256: string }).payloadSha256, createHash("sha256").update(on.body + " ").digest("hex")); +}); + +test("the examples/http-client.mjs verifyEvent port accepts Ship's real delivery, dedupes, and refuses tampering", async () => { + // @ts-expect-error plain .mjs example with no declarations + const client = (await import("../examples/http-client.mjs")) as { verifyEvent: Function; EventDedupe: new () => unknown }; + const on = await deliver({ eventEnvelope: true }); + const headers = Object.fromEntries(Object.entries(on.headers).map(([k, v]) => [k.toLowerCase(), v])); + const dedupe = new client.EventDedupe(); + const o = { nowMs: 1_790_000_000_000, dedupe }; + const first = client.verifyEvent(headers, on.body, "k", o); + assert.ok(first.ok && !first.duplicate && first.event.cursor === 7); + assert.ok(client.verifyEvent(headers, on.body, "k", o).duplicate, "retry is a duplicate"); + const fresh = { nowMs: 1_790_000_000_000 }; + assert.equal(client.verifyEvent(headers, on.body + " ", "k", fresh).reason, "body-mismatch"); + assert.equal(client.verifyEvent(headers, on.body, "wrong", fresh).reason, "bad-signature"); + assert.equal(client.verifyEvent({ ...headers, "x-teploy-event": headers["x-teploy-event"]!.replace('"cursor":7', '"cursor":9') }, on.body, "k", fresh).reason, "bad-signature"); + assert.equal(client.verifyEvent(headers, on.body, "k", { nowMs: 1_790_000_000_000 + 10 * 60_000 }).reason, "expired"); + const off = await deliver({ eventEnvelope: false }); + assert.equal(client.verifyEvent(Object.fromEntries(Object.entries(off.headers).map(([k, v]) => [k.toLowerCase(), v])), off.body, "k", fresh).reason, "no-envelope"); +}); + +test("envelope is omitted when it cannot be honest: no secret, or no event_seq", async () => { + const noSeq = await deliver({ eventEnvelope: true, event: { runId: "run-2", status: "failed" } }); + assert.equal(noSeq.headers["X-Teploy-Event"], undefined); + const noSecret = await deliver({ eventEnvelope: true, secret: "" }); + assert.equal(noSecret.headers["X-Teploy-Event"], undefined); + assert.equal(noSecret.headers["X-Teploy-Signature"], undefined); +}); + +test("SHIP_EVENT_ENVELOPE=on turns the headers on through the environment", async () => { + process.env.SHIP_EVENT_ENVELOPE = "on"; + assert.ok((await deliver({})).headers["X-Teploy-Event"] !== undefined); +}); + +// ------------------------------------------------------------ tool shadow + +const OK: ExecResult = { exitCode: 0, stdout: "ok", stderr: "", timedOut: false, truncated: false }; +function fakeExecutor(): AgentExecutor & { puts: string[] } { + const puts: string[] = []; + return { + puts, + async exec() { return OK; }, + async putFile(path: string) { puts.push(path); }, + async getFile() { return new Uint8Array(); }, + async destroy() {}, + } as unknown as AgentExecutor & { puts: string[] }; +} + +async function setup(m: unknown): Promise<{ dir: string; shadow: string }> { + const dir = await mkdtemp(join(tmpdir(), "tm-shadow-")); + await writeFile(join(dir, "manifest.json"), typeof m === "string" ? m : JSON.stringify(m)); + process.env.SHIP_TOOL_MANIFEST_FILE = join(dir, "manifest.json"); + process.env.SHIP_TOOL_MANIFEST_SHADOW_FILE = join(dir, "shadow.jsonl"); + return { dir, shadow: join(dir, "shadow.jsonl") }; +} + +test("shadow off (default): executeAction records nothing, even with a manifest configured", async () => { + const { dir, shadow } = await setup(manifest([tool("bash", ["read"])])); + try { + const r = await executeAction(fakeExecutor(), { kind: "bash", code: "rm -rf x" }); + assert.deepEqual(r, OK); + await observeToolCall({ kind: "bash", code: "x" }); + assert.deepEqual(await readdir(dir), ["manifest.json"], "no shadow file was created"); + process.env.SHIP_TOOL_MANIFEST = "off"; + await observeToolCall({ kind: "bash", code: "x" }); + assert.deepEqual(await readdir(dir), ["manifest.json"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shadow on the real executeAction path: findings are logged, the call is never blocked or altered", async () => { + // bash declared read-only; edit declared read-only; create and python undeclared. + const { dir, shadow } = await setup(manifest([tool("bash", ["read"]), tool("edit", ["read"])])); + try { + process.env.SHIP_TOOL_MANIFEST = "shadow"; + const ex = fakeExecutor(); + const outputs = [ + await executeAction(ex, { kind: "bash", code: "make" }), + await executeAction(ex, { kind: "create", file: "a.txt", content: "hi" }), + ]; + assert.equal(outputs[0], OK, "result object returned untouched"); + assert.match(outputs[1]!.stdout, /created a\.txt/, "the create still happened"); + assert.deepEqual(ex.puts, ["a.txt"]); + await new Promise((r) => setTimeout(r, 50)); + const recs = await readShadowRecords(shadow); + const kinds = recs.map((r) => (r.kind === "finding" ? `${r.tool}:${r.finding.kind}` : r.kind)).sort(); + // bash performed exec but declared only read; create is not declared at all. + assert.deepEqual(kinds, ["bash:undeclared-effect", "create:unknown-tool"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shadow: a conforming call records nothing; an effect the manifest declares is not a finding", async () => { + const { dir, shadow } = await setup(manifest([tool("bash", ["exec", "read"])])); + try { + process.env.SHIP_TOOL_MANIFEST = "shadow"; + await observeToolCall({ kind: "bash", code: "ls" }); + assert.deepEqual(await readShadowRecords(shadow), []); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shadow: write by a read-only declaration is classed write-when-read-only", async () => { + const { dir, shadow } = await setup(manifest([tool("edit", ["read"])])); + try { + process.env.SHIP_TOOL_MANIFEST = "shadow"; + await observeToolCall({ kind: "edit" }); + const recs = await readShadowRecords(shadow); + assert.equal(recs.length, 1); + assert.ok(recs[0]!.kind === "finding" && recs[0]!.finding.kind === "write-when-read-only"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shadow: no manifest file configured records nothing; a bad manifest records ONE error, not a finding per call", async () => { + const { dir, shadow } = await setup("{ not json"); + try { + process.env.SHIP_TOOL_MANIFEST = "shadow"; + delete process.env.SHIP_TOOL_MANIFEST_FILE; + await observeToolCall({ kind: "bash", code: "x" }); + assert.deepEqual(await readShadowRecords(shadow), []); + process.env.SHIP_TOOL_MANIFEST_FILE = join(dir, "manifest.json"); + for (let i = 0; i < 3; i++) await observeToolCall({ kind: "bash", code: "x" }); + const recs = await readShadowRecords(shadow); + assert.equal(recs.length, 1); + assert.equal(recs[0]!.kind, "error"); + // invalid-but-parseable manifest (unknown field) is also an error, never trusted + resetToolManifestShadow(); + await writeFile(join(dir, "manifest.json"), JSON.stringify({ ...manifest([tool("bash", ["exec"])]), grants: ["all"] })); + await observeToolCall({ kind: "bash", code: "x" }); + const after = await readShadowRecords(shadow); + assert.equal(after.length, 2); + assert.match((after[1] as { error: string }).error, /unknown field "grants"/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("shadow cannot fail the call: an unwritable shadow path is swallowed", async () => { + const { dir } = await setup(manifest([tool("bash", ["read"])])); + try { + process.env.SHIP_TOOL_MANIFEST = "shadow"; + process.env.SHIP_TOOL_MANIFEST_SHADOW_FILE = join(dir, "manifest.json", "nope", "x.jsonl"); // parent is a file + assert.deepEqual(await executeAction(fakeExecutor(), { kind: "bash", code: "x" }), OK); + await observeToolCall({ kind: "bash", code: "x" }); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ------------------------------------------------------------ CLI dry run + +async function cli(args: string[]): Promise<{ code: number; stdout: string; stderr: string; home: string }> { + const home = await mkdtemp(join(tmpdir(), "tool-cli-home-")); + try { + const { stdout, stderr } = await run(process.execPath, [CLI, ...args], { env: { PATH: process.env.PATH ?? "", HOME: home, XDG_STATE_HOME: home }, timeout: 60_000 }); + return { code: 0, stdout, stderr, home: (await readdir(home)).join(",") }; + } catch (e) { + const err = e as { code?: number; stdout?: string; stderr?: string }; + return { code: err.code ?? 1, stdout: err.stdout ?? "", stderr: err.stderr ?? "", home: (await readdir(home)).join(",") }; + } finally { + await rm(home, { recursive: true, force: true }); + } +} + +test("tool validate: valid manifest, grant intersection, excess reported, nothing persisted", async () => { + const dir = await mkdtemp(join(tmpdir(), "tool-cli-")); + try { + const m = manifest([{ ...tool("list-issues", ["read", "network", "exec"]), permissions: { effects: ["read", "network", "exec"], hosts: ["api.forge.example", "evil.example"], scopes: ["issues:read", "admin"] } }]); + await writeFile(join(dir, "m.json"), JSON.stringify(m)); + const before = await readdir(dir); + const r = await cli(["tool", "validate", join(dir, "m.json"), "--grant", JSON.stringify({ effects: ["read", "network"], hosts: ["*.forge.example"], scopes: ["issues:read"] })]); + assert.equal(r.code, 0, r.stderr); + assert.match(r.stdout, /manifest: valid/); + assert.match(r.stdout, /granted: effects read,network {2}hosts api\.forge\.example {2}scopes issues:read/); + assert.match(r.stdout, /NOT granted: effects exec {2}hosts evil\.example {2}scopes admin/); + assert.match(r.stdout, /nothing was installed, stored or enforced/); + assert.deepEqual(await readdir(dir), before, "no files written next to the manifest"); + assert.equal(r.home, "", "no state written to the home/state directory"); + + const json = await cli(["tool", "validate", join(dir, "m.json"), "--json"]); + const report = JSON.parse(json.stdout); + assert.equal(report.dryRun, true); + assert.deepEqual(report.tools[0].granted, { effects: [], hosts: [], scopes: [] }, "no --grant grants nothing"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("tool validate: an invalid manifest exits 1 and grants nothing; a bad --grant is refused", async () => { + const dir = await mkdtemp(join(tmpdir(), "tool-cli-")); + try { + await writeFile(join(dir, "bad.json"), JSON.stringify({ ...manifest([tool("a", ["read"])]), sudo: true })); + const r = await cli(["tool", "validate", join(dir, "bad.json"), "--grant", '{"effects":["read"]}']); + assert.equal(r.code, 1); + assert.match(r.stdout, /INVALID/); + assert.match(r.stdout, /unknown field "sudo"/); + assert.doesNotMatch(r.stdout, /granted:/); + const g = await cli(["tool", "validate", join(dir, "bad.json"), "--grant", '{"effects":"read"}']); + assert.notEqual(g.code, 0); + assert.match(g.stderr, /bad --grant/); + const missing = await cli(["tool", "validate", join(dir, "nope.json")]); + assert.notEqual(missing.code, 0); + assert.match(missing.stderr, /cannot read manifest/); + assert.equal((await readFile(join(dir, "bad.json"), "utf8")).includes("sudo"), true); + } finally { + await rm(dir, { recursive: true, force: true }); + } +});