diff --git a/catalog/plugins.json b/catalog/plugins.json new file mode 100644 index 000000000..d7323b91e --- /dev/null +++ b/catalog/plugins.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "plugins": [ + { + "name": "babysitter", + "description": "Live-state PR babysitter: parallel review lenses, deterministic reconciliation, exact-head merge gate. Fail-closed: GitHub pull_request.ready_for_review, labeled, and unlabeled are not in the surface registry, so a manifest that declares them is refused plugin_event_unroutable until the relayfile adapter catalog grows.", + "source": { "owner": "AgentWorkforce", "repo": "flows", "path": "examples/babysitter" }, + "ref": "05c3dff138883322e80cb793b1f5a097ad510572", + "digest": "ae6af3335eb6d4e54559327acc1465419244b47911d8ff356850b61f6228d862", + "compat": { "surface": "^2.0.22", "sdk": "^2.0.22", "base": ["software-factory"] }, + "tier": "community", + "base": ["software-factory"] + } + ] +} diff --git a/docs/CLOUD.md b/docs/CLOUD.md index 575405efb..c48de401a 100644 --- a/docs/CLOUD.md +++ b/docs/CLOUD.md @@ -43,6 +43,17 @@ An authored `.flow.ts` takes `--input` exactly as a local direct run does (an existing JSON file, otherwise inline JSON), and travels as one self-contained source with its pinned Surface authority. +Flow-extension plugins declared in `flows.json` (and extra `--plugin ` on `flows deploy`) travel in the deploy/run body as `extensions[]`: +name, version, canonical ref, digest, manifest, and the plugin files (UTF-8 +or base64). The extensions field is capped at 2 MB separately from the 256 KB +source cap. Cloud must materialize them at `.flows/plugins/@sha256:/` +before the hosted CLI loads the source; until that Cloud slice lands, a +deployment that includes plugins is accepted by this CLI but not yet executed +as a composed graph on the hosted runner. Private repositories are +unsupported. `permissions.writes` is a reviewed declaration, unenforced +until gate 8. + ## Code sync ```sh diff --git a/docs/SURFACE.md b/docs/SURFACE.md index b1c7f62ac..ded2d041e 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -562,6 +562,74 @@ plugin code bundling/pinning, declarative-flow plugin preflight, and restart recovery of an interrupted plugin effect. Plugin effects currently inherit the internal authored executor's child-run lifecycle, not a resumable authored root. +### Flow extensions: schema 2, `kind: "flow-extension"` + +The same `flows-plugin.json` file carries a second kind. A **helper** plugin +(`kind` absent) extends `Ctx` with verbs and installs from npm as above. A +**flow extension** (`"schema": 2, "kind": "flow-extension"`) is a directory in +a public GitHub repository whose `entry` default-exports `flow()` and declares +what it will contribute to a base flow — `extends.handlers` (its `.on()` +pairs), `extends.hooks` (named points the base calls), `triggers` (validated +against the surface event registry, refused with `plugin_event_unroutable` +otherwise), `permissions` (integrations, harnesses, mcp, declared-but-unenforced +`writes`, a budget ceiling), `compat` (semver ranges for surface and sdk, and +the base flows it extends), and the same mandatory `preflight`. Validation is +`packages/sdk/src/flow-extension-manifest.ts`; the worked Babysitter manifest is +`testdata/plugins/extension-babysitter/flows-plugin.json`. + +```text +flows add github:/@# # or https://github.com///tree// +flows plugin list [--json] +flows plugin verify [--json] [--offline] +flows plugin remove [--json] +flows plugin update [--json] [--yes] [--to ] [] +``` + +`flows add` resolves the branch, tag, or commit to a 40-hex sha through +unauthenticated public GitHub reads (a private repository answers 404 and is +reported as `plugin_source_unresolved`), enumerates the tree at that commit — +refusing symlinks, submodules, traversal, a truncated listing, files over +256 KB, or plugins over 2 MB — downloads each blob pinned to the sha, checks +byte counts, and computes the content digest as the sha256 of the same +canonical `[{bytes,path,sha256}]` manifest a sealed bundle uses. The bytes are +materialized under `.flows/plugins/@sha256:/`; `flows.json.plugins` +gains the canonical `github:/@#` (a branch or tag is +never persisted); and `flows.lock.json` (version 2) records name, version, +source, digest, manifest hash, and the declaration order that will be the +composition order. `flows plugin verify` re-hashes the store against the lock +and, unless `--offline`, re-fetches the pinned commit; any difference is +`plugin_source_drift`, exit 2. + +**Composition.** `loadAuthoredFlow` (the path under `flows check`, `flows run`, +and the authored root) composes the project's extensions onto the base flow +(`packages/sdk/src/flow-extension-loader.ts`), in this fixed order: the +declaration and the lockfile must agree; the store is re-hashed against the +lock's digest and the manifest bytes against its manifest hash — nothing under +`.flows/plugins` is read as code before that passes; the manifest is validated +and its `compat` checked against the runtime and the base flow (`FlowHeader.version` +is matched when present; without it only `*` is satisfiable; a budget ceiling +above the base is `plugin_incompatible`); only then is the entry imported, its +handlers checked against the manifest's declared triggers (an entry cannot +subscribe to more than it declared), and appended **after** the base's own +handlers in lockfile order. Named `hooks` exports are matched to +`extends.hooks` and to the base header's `hooks` list; `f.hook` AND-composes +them in lock order. Nothing replaces, reorders, or widens a base handler, and +the base's definition object is untouched. `flows check` prints one `EXTENSION` +line per composed extension. Not composed by this release, and refused with +`plugin_unsupported` rather than ignored: an entry `use:` header, schedule +triggers, and gates; a generic `webhook(...)` handler is refused as +undeclared. Cloud deploy and hosted runs send composed extensions in the request body +(`extensions[]`, 2 MB cap, `--plugin` is send-only). Handler bodies still +execute nowhere (#301); what composition changes today is the declared +trigger set that `flows check`, requirements, and future dispatch read. + +GitHub `pull_request.ready_for_review`, `pull_request.labeled`, and +`pull_request.unlabeled` are **not** in the surface registry. The registry is +generated from the pinned relayfile adapter mappings (`scripts/generate-triggers.mjs`); +this repo cannot add those actions without an adapter-package change. A +Babysitter manifest that declares them is refused `plugin_event_unroutable` +until that upstream catalog grows. + ## 4. Build: the immutable bundle `flows build` seals a flow into a content-addressed, immutable bundle: canonical spec JSON, compiled TS with pinned deps, helper/plugin lockfile, assets, preflight declaration, identity signature — `flow@sha256:…`, pushed to a bucket/registry. `flows deploy` points a trigger at a digest; `flows run flow@sha256:…` executes from the bucket on any cell, no checkout. Preflight runs at build time for everything build-provable and again at deploy time for environment facts (credentials, workers, MCP servers). The working tree is for authoring; **production only ever runs digests.** diff --git a/examples/README.md b/examples/README.md index efc86de66..961b5fc09 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,14 @@ flow straight from this repo, shows its steps, and asks you to connect whatever Or from a checkout: `flows deploy --repo --on --approver `. +Install a plugin onto a base flow (Babysitter on Software Garden): + +```text +flows add github:AgentWorkforce/flows@#examples/babysitter +``` + +The plugin is recorded in `flows.json` / `flows.lock.json` and composed at load time. Hosted deploy accepts `--plugin ` as a send-only extra. Private repositories are unsupported. `permissions.writes` is declared, not enforced. + ## Gallery status **3 of 4 gallery entries pass; one is blocked.** The blocked entry fails diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index 43c37a8f4..91f778c18 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -29,7 +29,11 @@ const WORK = ".relayflow"; // test result, the exit code does. Skips honestly when there is nothing to run. const TEST = 'if [ -f package.json ] && node -e \'p=require("./package.json");process.exit(p.scripts&&p.scripts.test?0:1)\'; then npm ci --no-audit --no-fund && npm test; else echo "no test script; skipping"; fi'; -export default flow("software-factory", { budget: { dollars: 10, wallclock: "1h" } }, async (f, input) => { +export default flow("software-factory", { + version: "2.0.22", + hooks: ["pre-implement", "post-review", "merge-gate"], + budget: { dollars: 10, wallclock: "1h" }, +}, async (f, input) => { const { issue } = input; if (!issue?.title?.trim()) { // Parked, not canceled: a body cannot declare a kernel outcome, and the @@ -43,6 +47,11 @@ export default flow("software-factory", { budget: { dollars: 10, wallcloc // Fresh work dir, excluded from git, no leftover verdicts. await f.run(`rm -rf ${WORK} && mkdir -p ${WORK} && { grep -qxF '${WORK}/' .git/info/exclude 2>/dev/null || echo '${WORK}/' >> .git/info/exclude; }`); + if (!await f.hook("pre-implement", { title, issue })) { + await f.run("echo 'Stopped: pre-implement hook refused this ticket.' >&2"); + return f.done("declined"); + } + await f.agent("implementer", { cli: "claude", task: `Implement this ticket in the current repository, on the current branch, with regression tests. Commit as you go.\n` + @@ -62,14 +71,35 @@ export default flow("software-factory", { budget: { dollars: 10, wallcloc await f.run(TEST, { timeout: "15m" }); + if (!await f.hook("post-review", { title })) { + await f.run(`{ cat ${WORK}/summary.md; printf '\\n\\n## post-review: blocked\\n\\n'; } > ${WORK}/pr-body.md`); + await f.run("git add -A && (git diff --cached --quiet || git commit -qm 'Software factory: implementation and review fixes')"); + await f.run("git push --set-upstream origin HEAD"); + await f.run(`gh pr create --draft --title ${shellWord(`[blocked] ${title}`)} --body-file ${WORK}/pr-body.md`); + return f.done("step_failed"); + } + // Passed means exactly one verdict, and it is the pass marker. const verdict = await f.run(`if [ -f ${WORK}/review.passed ] && [ ! -f ${WORK}/review.blocked ]; then echo PASSED; else echo BLOCKED; fi`); await f.run("git add -A && (git diff --cached --quiet || git commit -qm 'Software factory: implementation and review fixes')"); await f.run("git push --set-upstream origin HEAD"); // Deterministic step, not an agent decision: the PR is opened either way, - // but a blocked review opens it as a draft with the findings attached. + // but a blocked review or a false merge-gate opens it as a draft. if (verdict.trim() === "PASSED") { + const origin = (await f.run("git remote get-url origin")).trim(); + const headSha = (await f.run("git rev-parse HEAD")).trim(); + const matched = /github\.com[:/]([^/]+)\/([^/]+?)(?:\.git)?$/.exec(origin); + const allowed = await f.hook("merge-gate", { + owner: matched?.[1] ?? "", + repo: matched?.[2] ?? "", + headSha, + }); + if (!allowed) { + await f.run(`{ cat ${WORK}/summary.md; printf '\\n\\n## merge-gate: blocked\\n\\n'; } > ${WORK}/pr-body.md`); + await f.run(`gh pr create --draft --title ${shellWord(`[blocked] ${title}`)} --body-file ${WORK}/pr-body.md`); + return f.done("step_failed"); + } await f.run(`gh pr create --title ${shellWord(title)} --body-file ${WORK}/summary.md`); return f.done("success"); } diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 965ad117b..bbd66d05e 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -44,6 +44,8 @@ import { } from './authored-flow-operation.js'; import { AuthoredFlowLifecycle } from './authored-flow-lifecycle.js'; import { JournalClient } from './journal-client.js'; +import { createHookEvaluator } from './authored-hooks.js'; +import { probeFlowExtension, type LoadedFlowExtension } from './flow-extension-loader.js'; import type { CompletionReason as ProtocolCompletionReason, RunCompletionReason as ProtocolRunCompletionReason, @@ -148,6 +150,8 @@ export interface ExecuteAuthoredFlowOptions { readonly localAgentStream?: string; /** Durable kernel root that owns this body's child admission identities. */ readonly rootRunId?: string; + /** Installed flow-extension plugins, in lock order, so `f.hook` can AND-compose them. */ + readonly extensions?: readonly LoadedFlowExtension[]; } export async function executeAuthoredFlow( @@ -171,7 +175,7 @@ export async function executeAuthoredFlow( ...(options.onWait !== undefined ? { onWait: options.onWait } : {}), }; const definition = getDefinition(handle); - const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget' && key !== 'memory'); + const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget' && key !== 'memory' && key !== 'version' && key !== 'hooks'); if (definition.header.tools && Object.keys(definition.header.tools).some(key => !['mcp', ...helperProviders.map(p => p.namespace)].includes(key))) headerFields.push('tools'); if (definition.header.tools?.relayfile !== undefined) headerFields.push('tools.relayfile'); const helperPreflight = checkSlackHelpers(definition); @@ -188,6 +192,9 @@ export async function executeAuthoredFlow( const checkedMcp = await checkMcpHeader(definition, flowPath); if (!checkedMcp.report.ok) throw new McpPreflightError(checkedMcp.report); + for (const extension of options.extensions ?? []) { + if (extension.manifest !== undefined) await probeFlowExtension(extension.manifest); + } const budget = new AuthoredBudget(definition.header.budget); if (definition.header.memory?.agent === true) { @@ -349,6 +356,17 @@ export async function executeAuthoredFlow( )); } + const evaluateHook = createHookEvaluator({ + journal, + ...(options.rootRunId === undefined ? {} : { rootRunId: options.rootRunId }), + flowName: definition.name, + declared: definition.header?.hooks ?? [], + extensions: options.extensions ?? [], + peekStep: () => nextStep, + restoreStep: (step) => { nextStep = step; }, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + const context: Ctx = { ...createHelpers((call: HelperCall): Step => { const verb = `${call.provider}.${call.verb}`; @@ -477,6 +495,23 @@ export async function executeAuthoredFlow( assertOperationAllowed('dispatch', definition.name, requestedCompletion); throw unsupportedVerb('dispatch'); }, + hook(name, input) { + assertOperationAllowed('hook', definition.name, requestedCompletion); + const id = `hook-${nextStep++}`; + const snapshot = snapshotJsonValue(input, 'f.hook input'); + return trackStep(authoredSteps, new AuthoredFlowOperation( + id, 'hook', + () => assertOperationAllowed('hook', definition.name, requestedCompletion), + async () => { + const verdict = await evaluateHook(id, name, snapshot, context); + const record = { hook: name, step: id, verdict: verdict ? 'pass' : 'fail' }; + const literal = `'${JSON.stringify(record).replaceAll("'", "'\\''")}'`; + await observeStep(id, 'deterministic', () => lowerDeterministic(id, `printf '%s' ${literal}`, false), options.onProgress); + return verdict; + }, + lifecycle, + )); + }, done(reason) { if (!isSurfaceFlowCompletionReason(reason)) { throw new AuthoredFlowExecutionError( diff --git a/packages/sdk/src/authored-flow-loader.ts b/packages/sdk/src/authored-flow-loader.ts index 04dd45aca..5a7db01a4 100644 --- a/packages/sdk/src/authored-flow-loader.ts +++ b/packages/sdk/src/authored-flow-loader.ts @@ -9,6 +9,9 @@ import { type AuthoredFlowDefinition, type FlowHandle, } from './authored-flow.js'; +import { canonicalize } from './canonical.js'; +import { composeDefinition, loadFlowExtensions, parseHooksExport, type ImportedFlow, type LoadedFlowExtension } from './flow-extension-loader.js'; +import type { RuntimeVersions } from './flow-extension-compat.js'; export class AuthoredFlowLoadError extends Error { constructor(message: string, readonly kind: 'invalid_spec' | 'use_not_found' | 'use_invalid' | 'use_cycle' = 'invalid_spec') { @@ -40,8 +43,20 @@ export interface LoadedAuthoredFlow { readonly getDefinition: GetFlowDefinition; /** Exact Surface package/runtime that owns the handle's WeakMap identity. */ readonly surfaceAuthority: SurfaceModuleAuthority; - /** Dependency-first load order, each canonical absolute path appearing once. */ + /** + * Dependency-first load order, each canonical absolute path appearing once. + * Flow-extension entries follow the root, so a graph of one node still + * means "one self-contained source" to the hosted paths that require it. + */ readonly graph: readonly LoadedAuthoredFlowNode[]; + /** Schema-2 flow extensions composed onto the root, in lock order; empty when the project declares none. */ + readonly extensions: readonly LoadedFlowExtension[]; +} + +export interface LoadAuthoredFlowOptions { + /** `compose` (default) verifies and appends the project's flow extensions; `none` loads the root alone. */ + readonly extensions?: 'compose' | 'none'; + readonly versions?: RuntimeVersions; } export interface LoadedAuthoredFlowNode { @@ -53,7 +68,7 @@ export interface LoadedAuthoredFlowNode { } /** Import and validate a direct-run module without executing its authored body. */ -export async function loadAuthoredFlow(path: string): Promise { +export async function loadAuthoredFlow(path: string, options: LoadAuthoredFlowOptions = {}): Promise { const loaded = new Map(); const visiting = new Set(); async function visit(sourcePath: string, isRoot = false): Promise { @@ -98,11 +113,39 @@ export async function loadAuthoredFlow(path: string): Promise canonicalize(a) === canonicalize(b), + ...(options.versions === undefined ? {} : { versions: options.versions }), + }); + if (extensions.length === 0) { + return Object.freeze({ sourcePath: root.path, handle: root.handle, getDefinition: root.getDefinition, + surfaceAuthority: root.surfaceAuthority, graph: Object.freeze(graph), extensions }); + } + const composed = composeDefinition(baseDefinition, extensions); + const getDefinition: GetFlowDefinition = (handle: FlowHandle) => + (handle === root.handle ? composed : root.getDefinition(handle)) as AuthoredFlowDefinition; + for (const extension of extensions) { + // The node carries the accessor the entry's own surface copy handed back: + // the root's accessor answers for the root's WeakMap only. + graph.push(Object.freeze({ path: extension.entryPath, handle: extension.handle, getDefinition: extension.getDefinition, + surfaceAuthority: root.surfaceAuthority, use: Object.freeze([]) })); + } + return Object.freeze({ sourcePath: root.path, handle: root.handle, getDefinition, + surfaceAuthority: root.surfaceAuthority, graph: Object.freeze(graph), extensions }); } -async function importAuthoredFlow(path: string): Promise> { +async function importAuthoredFlow(path: string): Promise> { const absolutePath = resolve(path); try { accessSync(absolutePath, constants.R_OK); @@ -128,7 +171,7 @@ async function importAuthoredFlow(path: string): Promise number; + readonly restoreStep?: (step: number) => void; + readonly signal?: AbortSignal; +}): (id: string, name: string, input: unknown, context: Ctx) => Promise { + let recorded: Promise> | undefined; + + async function load(): Promise> { + const verdicts = new Map(); + if (options.rootRunId === undefined) return verdicts; + let offset = 0; + for (;;) { + const page = await options.journal.streamRead(options.rootRunId, HOOK_STREAM, offset, 1000); + for (const message of page.messages) { + const raw = (message as { message?: unknown }).message ?? message; + if (typeof raw !== 'object' || raw === null) continue; + const record = raw as HookRecord; + if (typeof record.hook !== 'string' || typeof record.step !== 'string') continue; + if (record.verdict !== 'pass' && record.verdict !== 'fail' && record.verdict !== 'noop') continue; + if (record.plugin !== null && typeof record.plugin !== 'string') continue; + verdicts.set(recordKey(record), record); + } + if (page.messages.length === 0 || page.next_offset <= offset) break; + offset = page.next_offset; + } + return verdicts; + } + + async function lookup(key: string): Promise { + if (options.rootRunId === undefined) return undefined; + recorded ??= load(); + return (await recorded).get(key); + } + + async function append(record: HookRecord): Promise { + if (options.rootRunId === undefined) return; + recorded ??= load(); + await options.journal.streamAppend(options.rootRunId, HOOK_STREAM, record); + (await recorded).set(recordKey(record), record); + } + + return async (id, name, input, context) => { + if (!options.declared.includes(name)) { + throw new AuthoredFlowExecutionError( + 'unsupported_verb', + `hook "${name}" is not declared by flow "${options.flowName}"`, + ); + } + const impls = options.extensions.filter(extension => extension.hooks[name] !== undefined); + if (impls.length === 0) { + const existing = await lookup(`${id}:noop`); + const record = existing ?? { hook: name, step: id, plugin: null, verdict: 'noop' as const }; + if (existing === undefined) await append(record); + else if (existing.afterStep !== undefined) options.restoreStep?.(existing.afterStep); + return true; + } + for (const extension of impls) { + const key = `${id}:${extension.name}`; + let record = await lookup(key); + if (record === undefined) { + let verdict = false; + let because: string | undefined; + try { + verdict = await boundHook(extension.hooks[name]!(context, input), options.signal) === true; + } catch (error) { + // Cancellation is control-plane state, not a plugin verdict. The + // shared execution signal also stops tracked f.run/f.agent work; + // rethrow so a later resume may run the hook again instead of + // replaying a permanent decline caused by an interrupted attempt. + if (options.signal?.aborted) throw error; + verdict = false; + because = error instanceof Error ? error.message : String(error); + } + record = { + hook: name, step: id, plugin: extension.name, + verdict: verdict ? 'pass' : 'fail', + ...(because === undefined ? {} : { because }), + ...(options.peekStep === undefined ? {} : { afterStep: options.peekStep() }), + }; + await append(record); + } else if (record.afterStep !== undefined) { + options.restoreStep?.(record.afterStep); + } + if (record.verdict === 'fail') return false; + } + return true; + }; +} + +async function boundHook(run: Promise, signal?: AbortSignal): Promise { + // The parent run's wallclock budget owns the deadline. A second timer here + // would not be the signal captured by the hook's Ctx operations, allowing + // those operations to continue after this wrapper returned. + if (signal === undefined) return await run; + if (signal.aborted) throw signal.reason ?? new Error('hook cancelled'); + return await new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new Error('hook cancelled')); + signal.addEventListener('abort', onAbort, { once: true }); + run.then( + value => { signal.removeEventListener('abort', onAbort); resolve(value); }, + error => { signal.removeEventListener('abort', onAbort); reject(error); }, + ); + }); +} diff --git a/packages/sdk/src/authored-node-entry.ts b/packages/sdk/src/authored-node-entry.ts index 367840ebf..1909ea5f1 100644 --- a/packages/sdk/src/authored-node-entry.ts +++ b/packages/sdk/src/authored-node-entry.ts @@ -72,6 +72,7 @@ try { request.metadata.inputPresent ? request.metadata.input : undefined, { getDefinition: loaded.getDefinition, dataDir: request.dataDir, flowPath: request.metadata.flowPath, rootRunId: request.rootRunId, + extensions: loaded.extensions, localAgentStream: request.localAgentStream, signal: controller.signal, onProgress: event => send({ type: 'progress', event }), onWait: event => send({ type: 'wait', event }), diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index eb8adc78c..0b6513042 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -21,6 +21,12 @@ import { isSurfaceCompletionReason } from './authored-step-output.js'; const ROOT_KIND = 'relayflows.authored-root.v1'; +export interface AuthoredRootExtension { + readonly name: string; + readonly digest: string; + readonly ref: string; +} + export interface AuthoredRootMetadata { readonly kind: typeof ROOT_KIND; readonly flowName: string; @@ -28,6 +34,8 @@ export interface AuthoredRootMetadata { readonly sourceSha256: string; readonly surface: SurfaceModuleAuthority; readonly sources: readonly AuthoredRootSourceAuthority[]; + /** Plugin-level provenance in lock order; empty when the project declares none. */ + readonly extensions: readonly AuthoredRootExtension[]; readonly localAgentStream?: string; readonly inputPresent: boolean; readonly input?: unknown; @@ -68,6 +76,9 @@ export async function executeDurableAuthoredFlow( sourceSha256: sha256(source), surface: loaded.surfaceAuthority, sources: Object.freeze(sources), + extensions: Object.freeze((loaded.extensions ?? []).map(extension => Object.freeze({ + name: extension.name, digest: extension.digest, ref: extension.ref, + }))), ...(options.localAgentStream === undefined ? {} : { localAgentStream: options.localAgentStream }), inputPresent: input !== undefined, ...(input === undefined ? {} : { input: jsonSnapshot(input, 'authored root input') }), @@ -172,7 +183,7 @@ export async function readAuthoredRootMetadata( if (!isRootMetadata(value)) { throw new Error('authored root journal has malformed authority metadata'); } - return value; + return Object.freeze({ ...value, extensions: value.extensions ?? [] }); } async function driveRoot( @@ -203,6 +214,7 @@ async function driveRoot( flowPath: metadata.flowPath, localAgentStream: options.localAgentStream, rootRunId: dispatch.run_id, + extensions: loaded.extensions, ...options.lifecycle, signal: callerSignal === undefined ? rootSignal @@ -409,10 +421,19 @@ function isRootMetadata(value: unknown): value is AuthoredRootMetadata { && typeof surface.version === 'string' && /^[a-f0-9]{64}$/.test(surface.packageSha256 ?? '') && /^[a-f0-9]{64}$/.test(surface.runtimeSha256 ?? '') && Array.isArray(sources) && sources.length > 0 && sources.every(isRootSourceAuthority) + && (root.extensions === undefined || (Array.isArray(root.extensions) && root.extensions.every(isRootExtension))) && (root.localAgentStream === undefined || /^local-agent-[a-f0-9-]+$/.test(root.localAgentStream)); } +function isRootExtension(value: unknown): value is AuthoredRootExtension { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const extension = value as Partial; + return typeof extension.name === 'string' && extension.name.length > 0 + && /^[a-f0-9]{64}$/.test(extension.digest ?? '') + && typeof extension.ref === 'string' && extension.ref.startsWith('github:'); +} + function isRootSourceAuthority(value: unknown): value is AuthoredRootSourceAuthority { if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; const source = value as Partial; diff --git a/packages/sdk/src/authored-source-authority.ts b/packages/sdk/src/authored-source-authority.ts index ec045465e..a543ce0c9 100644 --- a/packages/sdk/src/authored-source-authority.ts +++ b/packages/sdk/src/authored-source-authority.ts @@ -35,6 +35,12 @@ export async function loadPinnedAuthoredSource(metadata: AuthoredRootMetadata, i if (canonicalize(loadedSources) !== canonicalize(metadata.sources)) { throw new Error('authored root declared source graph authority mismatch'); } + const loadedExtensions = (loaded.extensions ?? []).map(extension => ({ + name: extension.name, digest: extension.digest, ref: extension.ref, + })); + if (canonicalize(loadedExtensions) !== canonicalize(metadata.extensions ?? [])) { + throw new Error('authored root declared extension authority mismatch'); + } return loaded; } diff --git a/packages/sdk/src/bundle-extensions.ts b/packages/sdk/src/bundle-extensions.ts new file mode 100644 index 000000000..d008141fc --- /dev/null +++ b/packages/sdk/src/bundle-extensions.ts @@ -0,0 +1,85 @@ +import { readFile, readdir } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import type { BundleFile } from './bundle.js'; +import { safePath, sha256 } from './bundle.js'; +import { findPluginProject } from './plugin-loader.js'; +import { parsePluginLock, readPluginLock, reconcileDeclaredExtensions, type PluginLock } from './plugin-lock.js'; +import { PluginError } from './plugin-manifest.js'; +import { pluginStoreDirectory, readStoredPluginFiles, verifyStoredPlugin } from './plugin-store.js'; + +const EMPTY_LOCK: PluginLock = Object.freeze({ version: 2, plugins: Object.freeze([]) }); + +/** + * Plugin provenance and bytes to put in a sealed bundle: `lockfile.json` is + * the same v2 lock the project records, and each extension's materialized + * files land under `plugins//…`. No project / no github: plugins → + * `{ version: 2, plugins: [] }` and no files. Fail closed on lock/store drift. + */ +export async function collectBundleExtensions(flowPath: string): Promise<{ lock: PluginLock; files: BundleFile[] }> { + const root = findPluginProject(dirname(flowPath)); + if (root === undefined) return { lock: EMPTY_LOCK, files: [] }; + const locked = reconcileDeclaredExtensions(root); + const files: BundleFile[] = []; + for (const { entry } of locked) { + const stored = await readStoredPluginFiles(pluginStoreDirectory(root, entry.name, entry.digest), entry.digest); + for (const file of stored) files.push({ path: `plugins/${entry.name}/${file.path}`, data: file.data }); + } + return { lock: readPluginLock(root), files }; +} + +/** + * After `verifyBundle` has hashed every payload file, check that `lockfile.json` + * is a v2 plugin lock (or a legacy v1 `{version:1, adapters}` with no plugins) + * and that each locked extension's store digest matches `plugins//manifest.json`. + */ +export async function verifyBundlePluginLock(bundle: string): Promise { + let parsed: unknown; + try { parsed = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); } + catch { throw new Error('lockfile.json: not valid JSON'); } + if (isLegacyV1Lock(parsed) || isLegacyNpmLock(parsed)) { + await assertNoLegacyPluginPayload(bundle); + return; + } + let lock: PluginLock; + try { lock = parsePluginLock(parsed); } + catch (error) { + throw new Error(error instanceof PluginError ? error.message : 'lockfile.json: not a plugin lock'); + } + for (const entry of lock.plugins) { + if (!safePath(entry.name) || entry.name.includes('/')) { + throw new Error(`lockfile.json: plugin name ${entry.name} is not a safe path component`); + } + const directory = join(bundle, 'plugins', entry.name); + await verifyStoredPlugin(directory, entry.digest); + let pluginManifest: Buffer; + try { pluginManifest = await readFile(join(directory, 'flows-plugin.json')); } + catch { throw new Error(`lockfile.json: plugin ${entry.name} is missing plugins/${entry.name}/flows-plugin.json`); } + if (sha256(pluginManifest) !== entry.manifestSha256) { + throw new Error(`lockfile.json: plugin ${entry.name} flows-plugin.json does not match the lockfile manifest hash`); + } + } +} + +async function assertNoLegacyPluginPayload(bundle: string): Promise { + let entries: string[]; + try { entries = await readdir(join(bundle, 'plugins')); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + if (entries.length > 0) { + throw new Error('lockfile.json: legacy locks cannot authenticate plugins/ payloads'); + } +} + +function isLegacyV1Lock(value: unknown): boolean { + return typeof value === 'object' && value !== null && !Array.isArray(value) + && (value as { version?: unknown }).version === 1 + && Array.isArray((value as { adapters?: unknown }).adapters); +} + +function isLegacyNpmLock(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const lock = value as { lockfileVersion?: unknown; packages?: unknown }; + return (lock.lockfileVersion === 2 || lock.lockfileVersion === 3) && typeof lock.packages === 'object' && lock.packages !== null; +} diff --git a/packages/sdk/src/bundle-typescript.ts b/packages/sdk/src/bundle-typescript.ts index 8164c1a07..4014287bf 100644 --- a/packages/sdk/src/bundle-typescript.ts +++ b/packages/sdk/src/bundle-typescript.ts @@ -60,7 +60,7 @@ process.stdout.write(JSON.stringify({ spec, authored })); '--no-compile-autoload-bunfig', '--outfile', executable, input], directory); const files: BundleFile[] = [ { path: 'flow', data: await readFile(executable) }, - { path: 'lockfile.json', data: canonicalize(lock) }, + { path: 'package-lock.json', data: canonicalize(lock) }, ]; return { spec, authored: inspected.authored === true, compiler, files }; } finally { await rm(staging, { recursive: true, force: true }); } diff --git a/packages/sdk/src/bundle.ts b/packages/sdk/src/bundle.ts index 470785f90..bbf986783 100644 --- a/packages/sdk/src/bundle.ts +++ b/packages/sdk/src/bundle.ts @@ -18,12 +18,24 @@ export function sha256(data: Uint8Array | string): string { return createHash('sha256').update(data).digest('hex'); } -function safePath(path: string): boolean { +/** A bundle-relative path: no empty, `.`, or `..` component, no backslash, NUL, or drive colon. */ +export function safePath(path: string): boolean { return path.length > 0 && !path.includes('\\') && !path.includes('\0') && path.split('/').every(part => part !== '' && part !== '.' && part !== '..') && !path.includes(':'); } +/** + * The canonical entry list whose sha256 is a payload's content digest. Shared + * by sealed flow bundles and materialized plugins so `@sha256:` means one + * thing everywhere: the digest of `[{bytes,path,sha256}]`, sorted by path. + */ +export function payloadManifest(files: readonly { path: string; data: Uint8Array }[]): string { + return canonicalize([...files] + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0) + .map(file => ({ path: file.path, sha256: sha256(file.data), bytes: file.data.length }))); +} + /** Manifest and identity are envelopes, excluded to avoid circular hashing. */ export async function sealBundle(options: BundleOptions): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(options.name)) { @@ -41,9 +53,7 @@ export async function sealBundle(options: BundleOptions): Promise { for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) { if (!paths.has(required)) throw new Error(`${required}: missing bundle file`); } - const manifest = canonicalize(files.map(file => ({ - path: file.path, sha256: sha256(file.data), bytes: file.data.length, - }))); + const manifest = payloadManifest(files); const digest = sha256(manifest); const out = resolve(options.out); const target = join(out, `${options.name}@sha256:${digest}`); diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index ed2ee8669..0bf944748 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -104,8 +104,8 @@ const LOCAL_EXECUTION_OPTIONS = [ export const CLI_VERBS = [ { name: 'add', - description: 'Install a helper plugin into this project', - args: [{ name: 'helper', description: 'Helper name or @flows/', required: true }], + description: 'Install a helper plugin, or a flow-extension plugin from a public GitHub repository, into this project', + args: [{ name: 'plugin', description: 'Helper name, @flows/, github:/@#, or a github.com tree URL', required: true }], variants: ['add'], }, { @@ -155,6 +155,7 @@ export const CLI_VERBS = [ { flags: '--agents ', description: 'Agent harnesses to allow, as claude[,codex]' }, { flags: '--name ', description: 'Name for the hosted listener' }, { flags: '--draft', description: 'Create the listener without activating it' }, + { flags: '--plugin ', description: 'Send-only GitHub flow-extension ref; repeatable. Does not write flows.json' }, NO_CONNECT_OPTION, JSON_OPTION, ], @@ -199,6 +200,35 @@ export const CLI_VERBS = [ options: [DATA_DIR_OPTION], variants: ['observer'], }, + { + name: 'plugin', + description: 'Inspect, remove, or update the flow-extension plugins recorded in flows.lock.json', + subcommands: [ + { name: 'list', description: 'List installed flow extensions in composition order', options: [JSON_OPTION] }, + { + name: 'verify', + description: 'Re-hash .flows/plugins against the lockfile and, unless --offline, against the pinned commit on GitHub', + options: [JSON_OPTION, { flags: '--offline', description: 'Skip the GitHub re-fetch; check only the local store against the lockfile' }], + }, + { + name: 'remove', + description: 'Drop a flow-extension plugin from flows.json, the lockfile, and the local store', + args: [{ name: 'name', description: 'Plugin name as recorded in the lockfile', required: true }], + options: [JSON_OPTION], + }, + { + name: 'update', + description: 'Re-resolve a flow-extension plugin, show the permissions/events/budget diff, and rewrite the lock with --yes', + args: [{ name: 'name', description: 'Plugin name; omit to update every installed flow extension', required: false }], + options: [ + JSON_OPTION, + { flags: '--to ', description: 'GitHub reference to resolve instead of the locked commit' }, + { flags: '--yes', description: 'Apply the update; without this flag the diff is printed and the lock is left unchanged' }, + ], + }, + ], + variants: ['plugin'], + }, { name: 'replay', description: 'Replay a finished run from its local journal', diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 75bfc1b1f..202a69a4a 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node import { addPlugin } from './cli/add.js'; +import { parsePluginArgs, runPluginCommand, type PluginArgs } from './cli/plugin.js'; import { watchCheck } from './cli-watch.js'; import { checkHelperBody } from './cli/check-helper-body.js'; import { describeFlowRequirements } from './flow-requirements.js'; @@ -64,6 +65,7 @@ type CliExitCode = 0 | 1 | 2 | 3; */ export type ParsedArgs = | { command: 'add'; value: string } + | PluginArgs | ReplayArgs | StatusArgs | BuildArgs @@ -92,9 +94,14 @@ export type ParsedArgs = const USAGE = [ 'Usage:', 'flows add ', + 'flows add ', + 'flows plugin list [--json]', + 'flows plugin verify [--json] [--offline]', + 'flows plugin remove [--json] ', + 'flows plugin update [--json] [--yes] [--to ] []', 'flows build [--out ] ', 'flows build --verify ', - 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--no-connect] [--json]', + 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--plugin ] [--no-connect] [--json]', 'flows deployments [--json]', 'flows undeploy [--json] ', 'flows schedule [--cron "" | --every ] [--tz ] [--input ] [--name ] [--no-connect] [--json]', @@ -192,6 +199,7 @@ export async function runCli( } if (parsed.command === 'add') return addPlugin(parsed.value, io); + if (parsed.command === 'plugin') return runPluginCommand(parsed, io); if (parsed.command === 'serve-webhook') { return withInterrupt(options.signal, (signal) => runServeWebhook(parsed, io, signal)); @@ -347,6 +355,8 @@ async function checkAuthoredFlowComposed(path: string): Promise<{ report: CheckR report: { ...mcp.report, ...(triggers?.report.schedules === undefined ? {} : { schedules: triggers.report.schedules }), + ...(triggers?.report.extensions === undefined ? {} : { extensions: triggers.report.extensions }), + ...(triggers?.report.hooks === undefined ? {} : { hooks: triggers.report.hooks }), // The authored definition sees helper flags, body use and `cli:` // declarations; the compiled view underneath knows only its steps. ...(triggers?.report.requirements === undefined ? {} : { requirements: triggers.report.requirements }), @@ -521,6 +531,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // parser, so the declared tree and the dispatched tree cannot drift apart. if (command === undefined || !CLI_VERB_NAMES.has(command)) return undefined; if (command === 'add') return args.length === 2 ? { command: 'add', value: args[1]! } : undefined; + if (command === 'plugin') return parsePluginArgs(args.slice(1)); if (command === 'replay') return parseReplayArgs(args.slice(1)); if (command === 'status') return parseStatusArgs(args.slice(1)); if (command === 'runs') return parseRunsArgs(args.slice(1)); @@ -937,6 +948,17 @@ function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { : `local: flows tick start --schedule-id ${schedule.scheduleId} --interval-ms ${schedule.intervalMs} --epoch-ms ${schedule.epochMs}`; io.stdout(`SCHEDULE handler ${schedule.handler} ${declared} -> flows.tick schedule_id ${schedule.scheduleId} [${local}]`); } + for (const extension of report.extensions ?? []) { + const hookList = (extension.hooks ?? []).length === 0 ? '' : `, hooks: ${extension.hooks!.join(', ')}`; + io.stdout(`EXTENSION ${extension.name}@${extension.version} ${extension.ref} sha256:${extension.digest} -> ${extension.handlers} handler(s) composed after the base flow${hookList}`); + } + if (report.hooks !== undefined) { + io.stdout(`HOOKS declared: ${report.hooks.declared.join(', ') || '(none)'}`); + for (const row of report.hooks.implementations) io.stdout(`HOOK ${row.hook} <- ${row.plugin}`); + for (const name of report.hooks.declared) { + if (!report.hooks.implementations.some(row => row.hook === name)) io.stdout(`HOOK ${name} <- (none)`); + } + } for (const resolution of report.resolutions) { const config = resolution.source === 'project' && report.projectConfigPath !== undefined ? ` (${report.projectConfigPath})` diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts new file mode 100644 index 000000000..c53ebeaf2 --- /dev/null +++ b/packages/sdk/src/cli/add-extension.ts @@ -0,0 +1,134 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { CliIo } from '../cli.js'; +import { sha256 } from '../bundle.js'; +import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha, type FetchLike, type FetchedPlugin } from '../plugin-github.js'; +import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, recoverFlowsAndLock, writeFlowsAndLock } from '../plugin-lock.js'; +import { findPluginProject } from '../plugin-loader.js'; +import { PluginError } from '../plugin-manifest.js'; +import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; +import { materializePlugin } from '../plugin-store.js'; + +export { assertCompatible, runtimeVersions }; + +/** Parse and validate the manifest inside a fetched plugin, checking that any self-declared source is the one it came from. */ +export function extensionManifestOf(plugin: FetchedPlugin): { manifest: FlowExtensionManifest; manifestSha256: string } { + const file = plugin.files.find(f => f.path === 'flows-plugin.json'); + if (file === undefined) throw new PluginError('plugin_manifest_missing', 'Plugin has no flows-plugin.json.'); + let input: unknown; + try { input = JSON.parse(file.data.toString('utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'flows-plugin.json is unreadable or invalid JSON.'); } + const manifest = validateFlowExtensionManifest(input); + const { source } = plugin; + if (manifest.source !== undefined && (manifest.source.owner !== source.owner || manifest.source.repo !== source.repo || manifest.source.path !== source.path + || (manifest.source.sha !== undefined && manifest.source.sha !== source.sha))) { + throw new PluginError('plugin_source_drift', `flows-plugin.json declares source ${manifest.source.owner}/${manifest.source.repo}#${manifest.source.path}, but it was fetched from ${source.owner}/${source.repo}#${source.path}.`); + } + if (!plugin.files.some(f => f.path === manifest.entry)) throw new PluginError('plugin_manifest_invalid', `entry ${manifest.entry} is not in the plugin.`); + return { manifest, manifestSha256: sha256(file.data) }; +} + +function eventKeys(manifest: FlowExtensionManifest): readonly string[] { + return manifest.triggers.map(t => `${t.provider} ${t.event}[${t.actions.join(',')}]`); +} + +function formatBudget(budget: FlowExtensionManifest['permissions']['budget']): string { + if (budget === undefined) return 'inherits base'; + return [budget.dollars === undefined ? '' : `$${budget.dollars}`, budget.wallclock ?? ''].filter(Boolean).join(' / ') || 'inherits base'; +} + +export function describeExtension(manifest: FlowExtensionManifest): string[] { + const p = manifest.permissions; + return [ + ` integrations: ${p.integrations.join(', ') || 'none'}; harnesses: ${p.harnesses.join(', ') || 'none'}; mcp: ${p.mcp.join(', ') || 'none'}`, + ` events: ${eventKeys(manifest).join('; ') || 'none'}`, + ` hooks: ${manifest.extends.hooks.join(', ') || 'none'}; handlers: ${manifest.extends.handlers ? 'yes' : 'no'}`, + ` writes (declared, unenforced): ${p.writes.join(', ') || 'none'}`, + ` budget: ${formatBudget(p.budget)}`, + ]; +} + +/** Permissions / events / budget (and version/hooks) changes between two manifests. */ +export function diffExtension(before: FlowExtensionManifest, after: FlowExtensionManifest): string[] { + const lines: string[] = []; + const change = (label: string, oldValue: string, newValue: string): void => { + if (oldValue !== newValue) lines.push(` ${label}: ${oldValue} → ${newValue}`); + }; + const setDiff = (label: string, oldList: readonly string[], newList: readonly string[]): void => { + const removed = oldList.filter(x => !newList.includes(x)); + const added = newList.filter(x => !oldList.includes(x)); + if (removed.length === 0 && added.length === 0) return; + lines.push(` ${label}: ${[...removed.map(x => `-${x}`), ...added.map(x => `+${x}`)].join(', ')}`); + }; + change('version', before.version, after.version); + setDiff('integrations', before.permissions.integrations, after.permissions.integrations); + setDiff('harnesses', before.permissions.harnesses, after.permissions.harnesses); + setDiff('mcp', before.permissions.mcp, after.permissions.mcp); + setDiff('writes', before.permissions.writes, after.permissions.writes); + setDiff('events', eventKeys(before), eventKeys(after)); + setDiff('hooks', before.extends.hooks, after.extends.hooks); + change('budget', formatBudget(before.permissions.budget), formatBudget(after.permissions.budget)); + if (lines.length === 0) lines.push(' (no permissions/events/budget changes)'); + return lines; +} + +export interface AddExtensionOptions { + cwd?: string; + fetch?: FetchLike; + now?: () => Date; + versions?: RuntimeVersions; +} + +/** + * `flows add github:/@#` — resolve the ref to a commit, + * fetch the plugin directory, validate its schema-2 manifest, materialize it + * under `.flows/plugins`, and record the canonical reference in `flows.json` + * plus the provenance in `flows.lock.json`. Runtime composition is a later + * slice: installing records a declaration, it does not enable execution. + */ +export async function addExtensionPlugin(input: string, io: CliIo, options: AddExtensionOptions = {}): Promise<0 | 2> { + try { + // Parse before touching the filesystem or the network: a malformed + // reference is refused offline, with the same code from any directory. + const requested = parsePluginSource(input); + const root = findPluginProject(options.cwd ?? process.cwd()); + if (!root) throw new PluginError('plugin_manifest_invalid', 'flows add requires a project with flows.json.'); + recoverFlowsAndLock(root); + const configPath = join(root, 'flows.json'); + const config = JSON.parse(readFileSync(configPath, 'utf8')); + if (!config || Array.isArray(config) || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every((p: unknown) => typeof p === 'string')))) throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); + const lock = readPluginLock(root); + const source = await resolveGithubSha(requested, options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const ref = canonicalPluginRef(source); + const declared: string[] = config.plugins ?? []; + for (const other of lock.plugins) { + if (other.name === manifest.name && canonicalPluginRef({ ...other.source, ref: other.source.sha }) !== ref) { + throw new PluginError('plugin_manifest_invalid', `Plugin ${manifest.name} is already installed from ${other.source.owner}/${other.source.repo}@${other.source.sha}; remove it before installing another source under the same name.`); + } + } + const { directory, digest } = await materializePlugin(root, manifest.name, fetched.files); + if (digest !== fetched.digest) throw new PluginError('plugin_source_drift', 'Materialized digest differs from the fetched digest.'); + const plugins = declared.includes(ref) ? declared : [...declared, ref]; + const next = lockWithPlugin(lock, plugins, { + name: manifest.name, version: manifest.version, + source: { host: 'github', owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }, + digest, manifestSha256, resolvedAt: (options.now ?? (() => new Date()))().toISOString(), + }); + writeFlowsAndLock(root, configPath, config, plugins, next); + io.stdout(`Added ${manifest.name}@${manifest.version} (flow-extension) from ${ref}`); + io.stdout(` digest sha256:${digest}`); + io.stdout(` materialized at ${directory}`); + for (const line of describeExtension(manifest)) io.stdout(line); + io.stdout(` recorded in flows.json and ${PLUGIN_LOCK_FILE}`); + return 0; + } catch (error) { + const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); + io.stderr(`REFUSED [${refusal.code}] ${refusal.message}`); + return 2; + } +} diff --git a/packages/sdk/src/cli/add.ts b/packages/sdk/src/cli/add.ts index 46912359e..5e6431dbd 100644 --- a/packages/sdk/src/cli/add.ts +++ b/packages/sdk/src/cli/add.ts @@ -3,16 +3,25 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { findPluginProject, probePlugin, readPlugin } from '../plugin-loader.js'; +import { recoverFlowsAndLock } from '../plugin-lock.js'; import { PluginError, pluginPackageName } from '../plugin-manifest.js'; +import { isGithubPluginRef } from '../plugin-source.js'; +import { addExtensionPlugin, type AddExtensionOptions } from './add-extension.js'; export async function addPlugin(name: string, io: CliIo, options: { cwd?: string; install?: (packageName: string, root: string) => void; + /** GitHub-sourced flow extensions only; ignored for helper packages. */ + extension?: Omit; } = {}): Promise<0 | 2> { + // A GitHub reference is a schema-2 flow extension; everything else is the + // helper-package path below, which this branch leaves exactly as it was. + if (isGithubPluginRef(name)) return addExtensionPlugin(name, io, { ...options.extension, ...(options.cwd === undefined ? {} : { cwd: options.cwd }) }); try { const packageName = pluginPackageName(name); const root = findPluginProject(options.cwd ?? process.cwd()); if (!root) throw new PluginError('plugin_manifest_invalid', 'flows add requires a project with flows.json.'); + recoverFlowsAndLock(root); const configPath = join(root, 'flows.json'); const config = JSON.parse(readFileSync(configPath, 'utf8')); if (!config || Array.isArray(config) || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every((p: unknown) => typeof p === 'string')))) throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); diff --git a/packages/sdk/src/cli/build.ts b/packages/sdk/src/cli/build.ts index 30ffa4988..61425cda7 100644 --- a/packages/sdk/src/cli/build.ts +++ b/packages/sdk/src/cli/build.ts @@ -2,6 +2,7 @@ import { readFile, lstat } from 'node:fs/promises'; import { dirname, extname, join, resolve } from 'node:path'; import { parse } from 'yaml'; import { sealBundle, verifyBundle, type BundleFile } from '../bundle.js'; +import { collectBundleExtensions, verifyBundlePluginLock } from '../bundle-extensions.js'; import { canonicalize } from '../canonical.js'; import { compileSpec, toKernelSpec } from '../compile.js'; import { preflight } from '../preflight.js'; @@ -60,6 +61,7 @@ export async function runBuild(args: BuildArgs, io: CliIo): Promise<0 | 2> { try { if (args.verify) { const digest = `sha256:${await verifyBundle(args.value)}`; + await verifyBundlePluginLock(args.value); // `--json` is declared on the verb, not on one of its forms: a verify // under it emits the same single object a `--json` consumer parses. if (args.json) io.stdout(JSON.stringify({ ok: true, verified: true, bundle: args.value, digest })); @@ -107,7 +109,7 @@ export async function buildFlow(path: string, out: string, warn: (line: string) files.push(...result.files); } else if (['.yaml', '.yml'].includes(extname(input))) { authoring = compileSpec(parse(await readFile(input, 'utf8'))); - files.push({ path: 'lockfile.json', data: canonicalize({ version: 1, adapters: [] }) }); + files.push({ path: 'lockfile.json', data: canonicalize({ version: 2, plugins: [] }) }); } else throw new Error('build expects a .yaml, .yml, or .ts flow'); // The current preflight API reports uncollected environment facts as @@ -148,7 +150,10 @@ export async function buildFlow(path: string, out: string, warn: (line: string) ...(authored ? { dynamicSteps: 'exported spec declaration checked; body was not executed during build' } : {}) }, }) }, ); - return sealBundle({ name: authoring.name ?? 'flow', out, files, + const extensions = await collectBundleExtensions(input); + const bundled = files.filter(file => file.path !== 'lockfile.json'); + bundled.push({ path: 'lockfile.json', data: canonicalize(extensions.lock) }, ...extensions.files); + return sealBundle({ name: authoring.name ?? 'flow', out, files: bundled, repo: await repositoryRoot(directory), warn }); } diff --git a/packages/sdk/src/cli/check-helper-body.ts b/packages/sdk/src/cli/check-helper-body.ts index febc5dcde..5a156400c 100644 --- a/packages/sdk/src/cli/check-helper-body.ts +++ b/packages/sdk/src/cli/check-helper-body.ts @@ -1,4 +1,5 @@ import { loadAuthoredFlow } from '../authored-flow-loader.js'; +import { PluginError } from '../plugin-manifest.js'; import { checkSlackHelpers } from '../slack-preflight.js'; import { inputFailureReport, type CheckExecution } from './check.js'; @@ -8,6 +9,9 @@ export async function checkHelperBody(path: string): Promise { const { handle, getDefinition } = await loadAuthoredFlow(path); return { report: { ...checkSlackHelpers(getDefinition(handle)), path } }; } catch (error) { + if (error instanceof PluginError) { + return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; + } return { report: inputFailureReport({ kind: 'invalid_spec', message: error instanceof Error ? error.message : 'Could not import authored flow.' }, path) }; } diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index bb64a1b37..d938fb7f0 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -1,10 +1,12 @@ import { dirname, resolve } from 'node:path'; import { loadAuthoredFlow, type LoadedAuthoredFlow } from '../authored-flow-loader.js'; +import { probeFlowExtension } from '../flow-extension-loader.js'; import { preflightWebhookTriggers } from '../preflight.js'; import { preflightProviderTriggers } from '../provider-trigger-contract.js'; import { scheduleLowering } from '../schedule-trigger.js'; import { checkSlackHelpers } from '../slack-preflight.js'; -import { flowRequirements } from '../flow-requirements.js'; +import { flowRequirements, mergeFlowExtensionRequirements } from '../flow-requirements.js'; +import { PluginError } from '../plugin-manifest.js'; import { inputFailureReport, readProjectConfig, type CheckReport } from './check.js'; /** @@ -22,6 +24,7 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const loaded = await loadAuthoredFlow(path); const definition = loaded.getDefinition(loaded.handle); const config = readProjectConfig(dirname(resolve(path))); + for (const extension of loaded.extensions ?? []) await probeFlowExtension(extension.manifest); const triggers = (definition.handlers ?? []).map(handler => handler.trigger); const triggerDiagnostics = preflightWebhookTriggers(triggers, config.executors); // Registration answers "may this inbox run here"; the provider contract @@ -40,6 +43,17 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const lowering = scheduleLowering(definition.name, trigger); return [{ handler, ...lowering }]; }); + // `?? []` tolerates the partial loader doubles the direct-run tests install. + const extensions = (loaded.extensions ?? []).map(extension => ({ + name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, + handlers: extension.handlers.length, + hooks: extension.manifest?.extends.hooks ?? [], + })); + const declaredHooks = definition.header?.hooks ?? []; + const implementations = (loaded.extensions ?? []).flatMap(extension => + (extension.manifest?.extends.hooks ?? []).map(hook => ({ hook, plugin: extension.name }))); + const hooks = declaredHooks.length > 0 || implementations.length > 0 + ? { declared: declaredHooks, implementations } : undefined; return { loaded, report: { @@ -47,11 +61,22 @@ export async function checkAuthoredTriggers(path: string): Promise<{ ok: !diagnostics.some(diagnostic => diagnostic.severity === 'refusal'), path, gates: [], resolutions: [], diagnostics, ...(schedules.length === 0 ? {} : { schedules }), - requirements: flowRequirements(definition, { projectCli: config.cli }), + ...(extensions.length === 0 ? {} : { extensions }), + ...(hooks === undefined ? {} : { hooks }), + requirements: mergeFlowExtensionRequirements( + flowRequirements(definition, { projectCli: config.cli }), + (loaded.extensions ?? []).map(extension => ({ + name: extension.name, permissions: extension.manifest.permissions, + })), + ), ...(config.path === undefined ? {} : { projectConfigPath: config.path }), }, }; } catch (error) { + // A flow-extension refusal keeps its own code (plugin_source_drift, + // plugin_incompatible, …): the operator needs to know which record + // disagreed, not that "the spec is invalid". + if (error instanceof PluginError) return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; return { report: inputFailureReport({ kind: typeof error === 'object' && error !== null && 'kind' in error && error.kind === 'config_invalid' ? 'config_invalid' : 'invalid_spec', diff --git a/packages/sdk/src/cli/check-typescript.ts b/packages/sdk/src/cli/check-typescript.ts index cf35cba3d..33b1de54a 100644 --- a/packages/sdk/src/cli/check-typescript.ts +++ b/packages/sdk/src/cli/check-typescript.ts @@ -2,6 +2,7 @@ import type { LoadedPlugin } from '../plugin-loader.js'; import { dirname, resolve } from 'node:path'; import type { AuthoredFlowDefinition } from '../authored-flow.js'; import { loadAuthoredFlow } from '../authored-flow-loader.js'; +import { PluginError } from '../plugin-manifest.js'; import { preflight } from '../preflight.js'; import { SPEC_SCHEMA_VERSION, type McpServerConfig } from '../spec.js'; import { inputFailureReport, readProjectConfig, type CheckReport } from './check.js'; @@ -18,6 +19,9 @@ export async function checkTypeScriptFlow(path: string): Promise<{ report: Check const { handle, getDefinition } = await loadAuthoredFlow(path); return await checkMcpHeader(getDefinition(handle), path); } catch (error) { + if (error instanceof PluginError) { + return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; + } return { report: inputFailureReport({ kind: 'invalid_spec', message: (error as Error).message }, path) }; } } @@ -28,7 +32,7 @@ export async function checkMcpHeader( path: string, ): Promise { const empty = { servers: Object.freeze({}), inventory: Object.freeze({}) }; - const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use']); + const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use', 'version', 'hooks']); const header = definition.header ?? {}; const unsupported = Object.keys(header).filter(key => !KNOWN_HEADER_FIELDS.has(key)); if (header.tools?.relayfile !== undefined) unsupported.push('tools.relayfile'); diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 22c362c9c..1ead84e21 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -61,9 +61,32 @@ export interface CheckReport { schedules?: ScheduleInspection[]; /** Integrations, harnesses and MCP servers the flow declares it needs (`flow-requirements.ts`). */ requirements?: FlowRequirements; + /** Schema-2 flow extensions composed onto the authored flow, in lock order (`flow-extension-loader.ts`). */ + extensions?: ExtensionInspection[]; + /** Base hook points and which plugins implement them. */ + hooks?: HookInspection; diagnostics: Array; } +export interface ExtensionInspection { + name: string; + version: string; + /** Canonical `github:/@#`. */ + ref: string; + digest: string; + /** How many `.on()` handlers it appends after the base flow's own. */ + handlers: number; + /** Hook names this extension implements, in manifest order. */ + hooks?: readonly string[]; +} + +export interface HookInspection { + /** Names the base flow header declares. */ + declared: readonly string[]; + /** Plugin implementations in lock order. */ + implementations: readonly { hook: string; plugin: string }[]; +} + export interface ScheduleInspection { /** Position among the flow's handlers, so two identical declarations stay distinct. */ handler: number; diff --git a/packages/sdk/src/cli/cloud-deploy.ts b/packages/sdk/src/cli/cloud-deploy.ts index 644ee602e..53bc522b7 100644 --- a/packages/sdk/src/cli/cloud-deploy.ts +++ b/packages/sdk/src/cli/cloud-deploy.ts @@ -19,6 +19,7 @@ export interface CloudDeployArgs { /** Refuse a missing integration instead of offering to connect it. */ noConnect: boolean; json: boolean; + plugins: string[]; } /** @@ -39,6 +40,7 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | let noConnect = false; let json = false; const on: string[] = []; + const plugins: string[] = []; for (let i = 0; i < args.length; i++) { const arg = args[i]!; if (arg === '--json') { @@ -63,6 +65,13 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | i += 1; continue; } + if (arg === '--plugin') { + const next = args[i + 1]; + if (next === undefined || next.startsWith('-')) return undefined; + plugins.push(next); + i += 1; + continue; + } if (arg === '--repo' || arg === '--approver' || arg === '--name' || arg === '--on') { const next = args[i + 1]; if (next === undefined || next.startsWith('-')) return undefined; @@ -78,7 +87,7 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | value = arg; } if (value === undefined || repo === undefined || on.length === 0) return undefined; - return { command: 'cloud-deploy', value, repo, on, approver, name, agents, draft, noConnect, json }; + return { command: 'cloud-deploy', value, repo, on, approver, name, agents, draft, noConnect, json, plugins }; } function describeSource(source: FlowTriggerSource): string { @@ -109,6 +118,7 @@ export async function runCloudDeployCli(args: CloudDeployArgs, io: CliIo): Promi ...(args.name === undefined ? {} : { name: args.name }), ...(agents === undefined ? {} : { agents }), ...(connect === undefined ? {} : { connect }), + ...(args.plugins.length === 0 ? {} : { plugins: args.plugins }), }); if (args.json) { io.stdout(JSON.stringify({ ok: true, ...deployment })); diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts new file mode 100644 index 000000000..c4e234718 --- /dev/null +++ b/packages/sdk/src/cli/plugin.ts @@ -0,0 +1,286 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { CliIo } from '../cli.js'; +import { diffExtension, extensionManifestOf } from './add-extension.js'; +import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha, type FetchLike } from '../plugin-github.js'; +import { + PLUGIN_LOCK_FILE, lockForDeclared, lockWithPlugin, lockedPlugins, readPluginLock, reconcileDeclaredExtensions, recoverFlowsAndLock, + writeFlowsAndLock, type PluginLock, type PluginLockEntry, +} from '../plugin-lock.js'; +import { findPluginProject } from '../plugin-loader.js'; +import { PluginError } from '../plugin-manifest.js'; +import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; +import { materializePlugin, pluginStoreDirectory, removeStoredPlugin, verifyStoredPlugin } from '../plugin-store.js'; + +export type PluginArgs = + | { command: 'plugin'; sub: 'list'; json: boolean } + | { command: 'plugin'; sub: 'verify'; json: boolean; offline: boolean } + | { command: 'plugin'; sub: 'remove'; json: boolean; name: string } + | { command: 'plugin'; sub: 'update'; json: boolean; yes: boolean; name: string | undefined; to: string | undefined }; + +/** `flows plugin list|verify|remove|update` */ +export function parsePluginArgs(args: readonly string[]): PluginArgs | undefined { + const [sub, ...rest] = args; + if (sub === 'list' || sub === 'verify') { + let json = false; + let offline = false; + for (const arg of rest) { + if (arg === '--json' && !json) json = true; + else if (arg === '--offline' && !offline && sub === 'verify') offline = true; + else return undefined; + } + return sub === 'list' ? { command: 'plugin', sub, json } : { command: 'plugin', sub, json, offline }; + } + if (sub === 'remove') { + let json = false; + let name: string | undefined; + for (const arg of rest) { + if (arg === '--json' && !json) json = true; + else if (arg.startsWith('-') || name !== undefined) return undefined; + else name = arg; + } + return name === undefined ? undefined : { command: 'plugin', sub: 'remove', json, name }; + } + if (sub !== 'update') return undefined; + let json = false; + let yes = false; + let to: string | undefined; + let name: string | undefined; + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]!; + if (arg === '--json' && !json) json = true; + else if (arg === '--yes' && !yes) yes = true; + else if (arg === '--to') { + const value = rest[i + 1]; + if (to !== undefined || value === undefined || value.startsWith('-')) return undefined; + to = value; + i += 1; + } else if (arg.startsWith('-') || name !== undefined) return undefined; + else name = arg; + } + return { command: 'plugin', sub: 'update', json, yes, name, to }; +} + +/** + * Cross-check the three records that must agree: `flows.json.plugins` + * (declaration), `flows.lock.json` (provenance), and `.flows/plugins` (bytes). + * With the network, the pinned commit is re-fetched and re-hashed too. + */ +export async function verifyPlugins(root: string, options: { offline: boolean; fetch?: FetchLike }): Promise<{ entry: PluginLockEntry; ref: string; directory: string; remote: 'verified' | 'skipped' }[]> { + const results = []; + for (const { ref, entry, source } of reconcileDeclaredExtensions(root)) { + const directory = pluginStoreDirectory(root, entry.name, entry.digest); + await verifyStoredPlugin(directory, entry.digest); + let remote: 'verified' | 'skipped' = 'skipped'; + if (!options.offline) { + const fetched = await fetchGithubPlugin(source, options.fetch); + if (fetched.digest !== entry.digest) throw new PluginError('plugin_source_drift', `${ref}: GitHub now serves digest ${fetched.digest}, lockfile has ${entry.digest}.`); + remote = 'verified'; + } + results.push({ entry, ref, directory, remote }); + } + return results; +} + +export interface PluginCommandOptions { + cwd?: string; + fetch?: FetchLike; + now?: () => Date; + versions?: RuntimeVersions; +} + +export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: PluginCommandOptions = {}): Promise<0 | 2> { + try { + const root = findPluginProject(options.cwd ?? process.cwd()); + if (!root) throw new PluginError('plugin_manifest_invalid', 'flows plugin requires a project with flows.json.'); + if (parsed.sub === 'list') { + const plugins = lockedPlugins(readPluginLock(root)); + if (parsed.json) { io.stdout(JSON.stringify({ plugins: plugins.map(p => ({ ...p.entry, ref: p.ref })) })); return 0; } + if (plugins.length === 0) { io.stdout('No flow-extension plugins installed.'); return 0; } + for (const { entry, ref } of plugins) io.stdout(`${entry.order}. ${entry.name}@${entry.version} ${ref} sha256:${entry.digest}`); + return 0; + } + if (parsed.sub === 'verify') { + const results = await verifyPlugins(root, { offline: parsed.offline, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) }); + if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: results.map(r => ({ name: r.entry.name, ref: r.ref, digest: r.entry.digest, remote: r.remote })) })); return 0; } + for (const r of results) io.stdout(`OK ${r.entry.name}@${r.entry.version} ${r.ref} local digest matches lockfile; remote ${r.remote}`); + if (results.length === 0) io.stdout('No flow-extension plugins to verify.'); + return 0; + } + if (parsed.sub === 'remove') return await removePlugin(root, parsed, io); + return await updatePlugins(root, parsed, io, options); + } catch (error) { + const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); + if (parsed.json) io.stdout(JSON.stringify({ ok: false, code: refusal.code, message: refusal.message })); + io.stderr(`REFUSED [${refusal.code}] ${refusal.message}`); + return 2; + } +} + +function readFlowsConfig(root: string): { path: string; config: Record; plugins: string[] } { + recoverFlowsAndLock(root); + const path = join(root, 'flows.json'); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(path, 'utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed) + || ((parsed as { plugins?: unknown }).plugins !== undefined + && (!Array.isArray((parsed as { plugins?: unknown }).plugins) + || !(parsed as { plugins: unknown[] }).plugins.every(p => typeof p === 'string')))) { + throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); + } + const config = parsed as Record; + return { path, config, plugins: [...((config.plugins as string[] | undefined) ?? [])] }; +} + +function storedManifest(directory: string): FlowExtensionManifest { + let raw: string; + try { raw = readFileSync(join(directory, 'flows-plugin.json'), 'utf8'); } + catch { throw new PluginError('plugin_source_drift', `${directory}: flows-plugin.json is missing.`); } + let input: unknown; + try { input = JSON.parse(raw); } + catch { throw new PluginError('plugin_manifest_invalid', `${directory}: flows-plugin.json is unreadable or invalid JSON.`); } + return validateFlowExtensionManifest(input); +} + +function storeReferenced(lock: PluginLock, name: string, digest: string): boolean { + return lock.plugins.some(p => p.name === name && p.digest === digest); +} + +async function dropUnreferencedStore(root: string, name: string, digest: string, lock: PluginLock): Promise { + if (storeReferenced(lock, name, digest)) return; + await removeStoredPlugin(pluginStoreDirectory(root, name, digest)); +} + +async function removePlugin(root: string, parsed: Extract, io: CliIo): Promise<0 | 2> { + const locked = lockedPlugins(readPluginLock(root)); + const match = locked.find(p => p.entry.name === parsed.name); + if (match === undefined) throw new PluginError('plugin_manifest_invalid', `No flow-extension plugin named ${parsed.name}.`); + const { path, config, plugins } = readFlowsConfig(root); + const nextDeclared = plugins.filter(ref => ref !== match.ref); + const nextLock = lockForDeclared(readPluginLock(root), nextDeclared); + writeFlowsAndLock(root, path, config, nextDeclared, nextLock); + await dropUnreferencedStore(root, match.entry.name, match.entry.digest, nextLock); + if (parsed.json) { + io.stdout(JSON.stringify({ ok: true, removed: { name: match.entry.name, ref: match.ref, digest: match.entry.digest } })); + return 0; + } + io.stdout(`Removed ${match.entry.name}@${match.entry.version} ${match.ref}`); + return 0; +} + +function replaceDeclaredRef(plugins: readonly string[], oldRef: string, newRef: string): string[] { + const without = plugins.filter(ref => ref !== oldRef); + if (without.includes(newRef)) return without; + const at = plugins.indexOf(oldRef); + const next = [...without]; + next.splice(at === -1 ? next.length : at, 0, newRef); + return next; +} + +async function updatePlugins( + root: string, + parsed: Extract, + io: CliIo, + options: PluginCommandOptions, +): Promise<0 | 2> { + if (parsed.to !== undefined && parsed.name === undefined) { + throw new PluginError('plugin_manifest_invalid', 'flows plugin update --to requires a plugin name.'); + } + const locked = lockedPlugins(readPluginLock(root)); + const targets = parsed.name === undefined ? locked : locked.filter(p => p.entry.name === parsed.name); + if (parsed.name !== undefined && targets.length === 0) { + throw new PluginError('plugin_manifest_invalid', `No flow-extension plugin named ${parsed.name}.`); + } + if (targets.length === 0) { + if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: [] })); return 0; } + io.stdout('No flow-extension plugins to update.'); + return 0; + } + + type Plan = { + current: (typeof locked)[number]; + ref: string; + digest: string; + manifest: FlowExtensionManifest; + manifestSha256: string; + source: { host: 'github'; owner: string; repo: string; sha: string; path: string }; + files: readonly { path: string; data: Uint8Array }[]; + diff: string[]; + changed: boolean; + }; + const plans: Plan[] = []; + for (const current of targets) { + const requested = parsed.to === undefined ? { ...current.source } : parsePluginSource(parsed.to); + const source = await resolveGithubSha(requested, options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + if (manifest.name !== current.entry.name) { + throw new PluginError('plugin_manifest_invalid', `Update of ${current.entry.name} resolved to plugin ${manifest.name}; remove it and add the new name instead.`); + } + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const directory = pluginStoreDirectory(root, current.entry.name, current.entry.digest); + const before = storedManifest(directory); + const ref = canonicalPluginRef(source); + const changed = ref !== current.ref || fetched.digest !== current.entry.digest || manifestSha256 !== current.entry.manifestSha256; + plans.push({ + current, ref, digest: fetched.digest, manifest, manifestSha256, + source: { host: 'github', owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }, + files: fetched.files, diff: diffExtension(before, manifest), changed, + }); + } + + const pending = plans.filter(p => p.changed); + const summary = plans.map(p => ({ + name: p.current.entry.name, from: p.current.ref, to: p.ref, digest: p.digest, changed: p.changed, diff: p.diff, + })); + if (!parsed.json) { + for (const plan of plans) { + if (!plan.changed) { + io.stdout(`${plan.current.entry.name} is already at ${plan.ref} sha256:${plan.digest}`); + continue; + } + io.stdout(`Update ${plan.current.entry.name} ${plan.current.ref} → ${plan.ref}`); + io.stdout(` digest sha256:${plan.current.entry.digest} → sha256:${plan.digest}`); + for (const line of plan.diff) io.stdout(line); + } + } + if (pending.length === 0) { + if (parsed.json) io.stdout(JSON.stringify({ ok: true, plugins: summary })); + return 0; + } + if (!parsed.yes) { + if (parsed.json) { + io.stdout(JSON.stringify({ + ok: false, applied: false, code: 'plugin_manifest_invalid', + message: 'Re-run with --yes to apply this update.', plugins: summary, + })); + io.stderr('REFUSED [plugin_manifest_invalid] Re-run with --yes to apply this update.'); + return 2; + } + throw new PluginError('plugin_manifest_invalid', 'Re-run with --yes to apply this update.'); + } + + let declared = readFlowsConfig(root).plugins; + let lock = readPluginLock(root); + const applied: { name: string; ref: string; digest: string }[] = []; + for (const plan of pending) { + const { directory, digest } = await materializePlugin(root, plan.manifest.name, plan.files); + if (digest !== plan.digest) throw new PluginError('plugin_source_drift', 'Materialized digest differs from the fetched digest.'); + declared = replaceDeclaredRef(declared, plan.current.ref, plan.ref); + lock = lockWithPlugin(lock, declared, { + name: plan.manifest.name, version: plan.manifest.version, source: plan.source, + digest, manifestSha256: plan.manifestSha256, + resolvedAt: (options.now ?? (() => new Date()))().toISOString(), + }); + const { path, config } = readFlowsConfig(root); + writeFlowsAndLock(root, path, config, declared, lock); + await dropUnreferencedStore(root, plan.current.entry.name, plan.current.entry.digest, lock); + applied.push({ name: plan.manifest.name, ref: plan.ref, digest }); + } + if (parsed.json) io.stdout(JSON.stringify({ ok: true, plugins: applied })); + else for (const item of applied) io.stdout(`Updated ${item.name} ${item.ref} sha256:${item.digest} recorded in flows.json and ${PLUGIN_LOCK_FILE}`); + return 0; +} diff --git a/packages/sdk/src/cloud-deploy.ts b/packages/sdk/src/cloud-deploy.ts index 01873427b..a83d3f94e 100644 --- a/packages/sdk/src/cloud-deploy.ts +++ b/packages/sdk/src/cloud-deploy.ts @@ -6,8 +6,11 @@ import { ensureIntegrationsConnected, type ConnectPrompt } from './cloud-connect import { CloudFlowError, cloudFetch, cloudRequest, isCloudRecord, type CloudConnectionOptions, } from './cloud-http.js'; -import { flowRequirements, type FlowRequirements } from './flow-requirements.js'; +import { + flowRequirements, mergeFlowExtensionRequirements, type FlowRequirements, +} from './flow-requirements.js'; import { readProjectConfig } from './cli/check.js'; +import { assertNoUseDependencies, collectExtensionSubmissions } from './flow-extension-submit.js'; /** * Hosted listener deployment: the CLI form of the agentrelay.com onboarding's @@ -67,6 +70,11 @@ export interface DeployToCloudInput { connect?: ConnectPrompt; /** Skip the pre-submission integration check entirely (Cloud still checks on activation). */ checkConnections?: boolean; + /** + * Extra GitHub plugin refs resolved send-only (same path as `flows add`, + * without writing flows.json). Project-declared extensions are always sent. + */ + plugins?: readonly string[]; } export const FLOW_AGENT_HARNESSES = ['claude', 'codex'] as const; @@ -170,10 +178,8 @@ export async function deployToCloud( throw new CloudFlowError('unsupported_source', `${input.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}`); } - if (loaded.graph.length !== 1) { - throw new CloudFlowError('unsupported_source', - 'Cloud deploys one self-contained .flow.ts source without use dependencies.'); - } + assertNoUseDependencies(loaded); + const extensions = await collectExtensionSubmissions(loaded, input.plugins ?? []); let projectCli: string | undefined; try { projectCli = readProjectConfig(dirname(resolve(input.path))).cli; @@ -204,9 +210,12 @@ export async function deployToCloud( } // Every launched run lands in the deployment's repository, so GitHub is // required even when no GitHub source wakes it. - const requirements = flowRequirements(definition, { - sources, repository: input.repository, ...(projectCli === undefined ? {} : { projectCli }), - }); + const requirements = mergeFlowExtensionRequirements( + flowRequirements(definition, { + sources, repository: input.repository, ...(projectCli === undefined ? {} : { projectCli }), + }), + extensions.map(extension => ({ name: extension.name, permissions: extension.manifest.permissions })), + ); // The declared harnesses become `inputs.agents`; one Cloud cannot run is // refused here rather than silently replaced by Claude, which activation // would then check while the deployed runs still call the declared CLI. @@ -237,6 +246,7 @@ export async function deployToCloud( inputs: { approver, agents }, repository: input.repository, sources, + ...(extensions.length === 0 ? {} : { extensions }), requirements: { integrations: requirements.integrations.map(i => i.provider), harnesses: requirements.harnesses, diff --git a/packages/sdk/src/cloud-run.ts b/packages/sdk/src/cloud-run.ts index 2223c9bf7..fb92081e9 100644 --- a/packages/sdk/src/cloud-run.ts +++ b/packages/sdk/src/cloud-run.ts @@ -9,6 +9,7 @@ import { CompileError, compileSpec, kernelToAuthoring, toKernelSpec } from './co import type { FlowSpec } from './spec.js'; import { snapshotJsonValue, type JsonValue } from './json-value.js'; import { loadAuthoredFlow, type SurfaceModuleAuthority } from './authored-flow-loader.js'; +import { assertNoUseDependencies, collectExtensionSubmissions, type FlowExtensionSubmission } from './flow-extension-submit.js'; import { CloudFlowError, cloudConnection, cloudFetch, cloudRequest, cloudRunId, isCloudRecord, type CloudConnectionOptions, @@ -61,6 +62,7 @@ export function cloudSubmissionBody(submission: CloudSubmission): Record { let spec: FlowSpec | undefined; - let authored: { source: string; authority: CloudAuthoredAuthority; name: string; schedules: ScheduleTriggerSource[] } | undefined; + let authored: { source: string; authority: CloudAuthoredAuthority; name: string; schedules: ScheduleTriggerSource[]; extensions: readonly FlowExtensionSubmission[] } | undefined; const inputPresent = Object.prototype.hasOwnProperty.call(options, 'input'); let authoredInput: JsonValue | undefined; try { @@ -122,14 +125,13 @@ export async function prepareCloudSubmission( `${flow.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}. ` + 'Run `flows check` on it from the same directory.'); } - if (loaded.graph.length !== 1) { - throw new CloudFlowError('unsupported_source', - 'Cloud authored submission currently accepts one self-contained .flow.ts source without use dependencies.'); - } + assertNoUseDependencies(loaded); + const extensions = await collectExtensionSubmissions(loaded); authored = { source, name: definition.name, schedules: definition.handlers.flatMap(h => h.trigger.kind === 'schedule' ? [h.trigger] : []), + extensions, authority: Object.freeze({ schemaVersion: 1, sourceSha256: createHash('sha256').update(bytes).digest('hex'), @@ -182,7 +184,15 @@ export async function prepareCloudSubmission( return { workflow: authored.source, fileType: 'ts', authoredAuthority: authored.authority, inputs: authoredInput, inputPresent: true, name: authored.name, schedules: authored.schedules, - specHash: createHash('sha256').update(canonicalize({ authority: authored.authority, input: authoredInput })).digest('hex'), + ...(authored.extensions.length === 0 ? {} : { extensions: authored.extensions }), + specHash: createHash('sha256').update(canonicalize({ + authority: authored.authority, input: authoredInput, + ...(authored.extensions.length === 0 ? {} : { + extensions: authored.extensions.map(extension => ({ + name: extension.name, digest: extension.digest, ref: extension.ref, + })), + }), + })).digest('hex'), }; } diff --git a/packages/sdk/src/flow-extension-compat.ts b/packages/sdk/src/flow-extension-compat.ts new file mode 100644 index 000000000..6c3101ccc --- /dev/null +++ b/packages/sdk/src/flow-extension-compat.ts @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs'; +import type { FlowExtensionManifest } from './flow-extension-manifest.js'; +import { PluginError } from './plugin-manifest.js'; +import { satisfiesRange } from './semver-range.js'; + +export interface RuntimeVersions { readonly sdk: string; readonly surface: string } + +/** The versions a plugin's `compat` is checked against: this SDK and the surface it pins. */ +export function runtimeVersions(): RuntimeVersions { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version: string; dependencies: Record }; + return { sdk: pkg.version, surface: pkg.dependencies['@relayflows/surface']! }; +} + +/** `compat.surface` / `compat.sdk` against the runtime: a miss is a refusal, never a warning. */ +export function assertCompatible(manifest: FlowExtensionManifest, versions: RuntimeVersions): void { + for (const [what, range, actual] of [['surface', manifest.compat.surface, versions.surface], ['sdk', manifest.compat.sdk, versions.sdk]] as const) { + if (!satisfiesRange(actual, range)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${what} ${range}; this runtime has ${actual}.`); + } +} + +/** + * `compat.base` against the flow being extended. `FlowHeader.version` is + * optional: a base without it matches only `"*"`. + */ +export function assertBaseCompatible(manifest: FlowExtensionManifest, base: { readonly name: string; readonly version?: string }): void { + const entry = manifest.compat.base.find(b => b.name === base.name); + if (entry === undefined) { + throw new PluginError('plugin_incompatible', `${manifest.name} extends ${manifest.compat.base.map(b => b.name).join(', ')}, not "${base.name}".`); + } + if (base.version === undefined) { + if (entry.version !== '*') throw new PluginError('plugin_incompatible', `${manifest.name} requires ${base.name} ${entry.version}, but the base flow declares no version; only "*" can be satisfied.`); + return; + } + if (!satisfiesRange(base.version, entry.version)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${base.name} ${entry.version}; the base flow is ${base.version}.`); +} diff --git a/packages/sdk/src/flow-extension-loader.ts b/packages/sdk/src/flow-extension-loader.ts new file mode 100644 index 000000000..14be52141 --- /dev/null +++ b/packages/sdk/src/flow-extension-loader.ts @@ -0,0 +1,217 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { Ctx } from '@relayflows/surface'; +import type { AuthoredFlowDefinition, FlowHandle } from './authored-flow.js'; +import { sha256 } from './bundle.js'; +import { assertBaseCompatible, assertCompatible, runtimeVersions, type RuntimeVersions } from './flow-extension-compat.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from './flow-extension-manifest.js'; +import { findPluginProject } from './plugin-loader.js'; +import { reconcileDeclaredExtensions, type PluginLockEntry } from './plugin-lock.js'; +import { PluginError } from './plugin-manifest.js'; +import { pluginStoreDirectory, verifyStoredPlugin } from './plugin-store.js'; + +/** + * Compose schema-2 flow extensions onto a base authored flow. + * + * Order of operations is the security argument, so it is fixed: + * 1. `flows.json.plugins` and `flows.lock.json` must agree (declaration ⇔ + * provenance, same order); + * 2. the materialized store is re-hashed against the lock's digest and the + * manifest bytes against the lock's manifest hash — nothing under + * `.flows/plugins` is read as code before this passes; + * 3. the manifest is validated, its compat checked against the runtime and + * the base flow, and anything this slice does not compose (hooks, `use`, + * schedule triggers, non-provider webhooks, gates) is refused; + * 4. only then is the entry imported, and its handlers are checked against + * the manifest's declared triggers — an entry cannot subscribe to more + * than it declared. + * Extensions compose after the base, in lock order; nothing replaces, + * reorders, or widens a base handler. + */ +type TriggerHandler = AuthoredFlowDefinition['handlers'][number]; + +export type FlowHook = (f: Ctx, input: unknown) => Promise; + +export interface LoadedFlowExtension { + readonly name: string; + readonly version: string; + readonly ref: string; + readonly digest: string; + readonly directory: string; + readonly entryPath: string; + readonly manifest: FlowExtensionManifest; + readonly handle: FlowHandle; + /** Bound to the surface copy the entry itself imported; the base's accessor cannot see this handle's WeakMap entry. */ + readonly getDefinition: ImportedFlow['getDefinition']; + readonly handlers: readonly TriggerHandler[]; + readonly hooks: Readonly>; +} + +export interface ImportedFlow { + readonly handle: FlowHandle; + readonly getDefinition: (handle: FlowHandle) => AuthoredFlowDefinition; + readonly surfaceAuthority: Authority; + readonly hooks: Readonly>; +} + +export interface LoadFlowExtensionsOptions { + readonly importFlow: (path: string) => Promise>; + readonly sameAuthority: (a: Authority, b: Authority) => boolean; + readonly versions?: RuntimeVersions; +} + +const EXTENSION_HEADER_FIELDS = new Set(['budget', 'tools']); +const WALLCLOCK_MS = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 } as const; + +function wallclockMs(value: string): number | undefined { + const match = /^(\d+)(ms|s|m|h|d)$/.exec(value); + if (!match) return undefined; + const unit = match[2] as keyof typeof WALLCLOCK_MS; + return Number(match[1]) * WALLCLOCK_MS[unit]; +} + +/** Credentials and servers declared on a flow-extension, probed before the base body starts. */ +export async function probeFlowExtension(manifest: FlowExtensionManifest, env: NodeJS.ProcessEnv = process.env): Promise { + for (const credential of manifest.preflight.credentials) { + if (!env[credential]?.trim()) throw new PluginError('plugin_credential_missing', `${manifest.name} requires ${credential}.`); + } + for (const server of manifest.preflight.servers) { + try { + const response = await fetch(server, { method: 'HEAD', signal: AbortSignal.timeout(5000) }); + await response.body?.cancel(); + if (!response.ok) throw new Error('unsuccessful response'); + } catch { throw new PluginError('plugin_server_unreachable', `${manifest.name} cannot reach ${server}.`); } + } +} + +function unsupported(name: string, what: string): never { + throw new PluginError('plugin_unsupported', `${name}: ${what} is not composed by this release.`); +} + +/** The `{provider, event, action?}` a provider subscription lowers to, or undefined for anything else. */ +function subscriptionOf(handler: TriggerHandler): { provider: string; event: string; action?: string } | undefined { + const trigger = handler.trigger; + if (trigger.kind !== 'webhook' || trigger.filter === undefined) return undefined; + const { provider, type, payload } = trigger.filter as { provider?: unknown; type?: unknown; payload?: unknown }; + if (typeof provider !== 'string' || provider !== trigger.name || typeof type !== 'string') return undefined; + const action = typeof payload === 'object' && payload !== null && !Array.isArray(payload) ? (payload as { action?: unknown }).action : undefined; + if (action !== undefined && typeof action !== 'string') return undefined; + return action === undefined ? { provider, event: type } : { provider, event: type, action }; +} + +function assertDeclaredSubscription(name: string, manifest: FlowExtensionManifest, handler: TriggerHandler, index: number): void { + const subscription = subscriptionOf(handler); + if (handler.trigger.kind === 'schedule') unsupported(name, `handler ${index} (a schedule trigger)`); + if (subscription === undefined) { + throw new PluginError('plugin_manifest_invalid', `${name}: handler ${index} is not a provider subscription; extension handlers must be provider triggers declared in flows-plugin.json.`); + } + const declared = manifest.triggers.find(t => t.provider === subscription.provider && t.event === subscription.event); + const covered = declared !== undefined && (subscription.action === undefined ? declared.actions.length === 0 : declared.actions.includes(subscription.action)); + if (!covered) { + const spelled = `${subscription.provider} ${subscription.event}${subscription.action === undefined ? '' : `.${subscription.action}`}`; + throw new PluginError('plugin_manifest_invalid', `${name}: handler ${index} subscribes to ${spelled}, which flows-plugin.json does not declare in triggers.`); + } +} + +async function loadOne( + root: string, lock: PluginLockEntry, ref: string, + base: { readonly definition: AuthoredFlowDefinition; readonly surfaceAuthority: Authority }, + options: LoadFlowExtensionsOptions, +): Promise { + const directory = pluginStoreDirectory(root, lock.name, lock.digest); + await verifyStoredPlugin(directory, lock.digest); + const manifestBytes = readFileSync(join(directory, 'flows-plugin.json')); + if (sha256(manifestBytes) !== lock.manifestSha256) throw new PluginError('plugin_source_drift', `${ref}: flows-plugin.json differs from the lockfile's manifest hash.`); + let input: unknown; + try { input = JSON.parse(manifestBytes.toString('utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', `${ref}: flows-plugin.json is not valid JSON.`); } + const manifest = validateFlowExtensionManifest(input); + if (manifest.name !== lock.name || manifest.version !== lock.version) throw new PluginError('plugin_source_drift', `${ref}: manifest names ${manifest.name}@${manifest.version}, lockfile has ${lock.name}@${lock.version}.`); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + assertBaseCompatible(manifest, { name: base.definition.name, version: base.definition.header.version }); + const baseBudget = base.definition.header.budget; + const ceiling = manifest.permissions.budget; + if (ceiling?.dollars !== undefined && typeof baseBudget === 'object' && baseBudget.dollars !== undefined && ceiling.dollars > baseBudget.dollars) { + throw new PluginError('plugin_incompatible', `${manifest.name} declares a $${ceiling.dollars} budget ceiling above the base flow's $${baseBudget.dollars}.`); + } + if (ceiling?.wallclock !== undefined && typeof baseBudget === 'object' && baseBudget.wallclock !== undefined) { + const pluginMs = wallclockMs(ceiling.wallclock); + const baseMs = wallclockMs(baseBudget.wallclock); + if (pluginMs !== undefined && baseMs !== undefined && pluginMs > baseMs) { + throw new PluginError('plugin_incompatible', `${manifest.name} declares a ${ceiling.wallclock} wallclock ceiling above the base flow's ${baseBudget.wallclock}.`); + } + } + const entryPath = join(directory, manifest.entry); + let imported: ImportedFlow; + try { imported = await options.importFlow(entryPath); } + catch (error) { throw new PluginError('plugin_manifest_invalid', `${ref}: entry ${manifest.entry} did not load: ${error instanceof Error ? error.message : String(error)}`); } + if (!options.sameAuthority(imported.surfaceAuthority, base.surfaceAuthority)) { + throw new PluginError('plugin_incompatible', `${manifest.name}: entry resolves a different @relayflows/surface than the base flow.`); + } + const definition = imported.getDefinition(imported.handle); + const foreign = Object.keys(definition.header).filter(key => !EXTENSION_HEADER_FIELDS.has(key)); + if (foreign.length > 0) unsupported(manifest.name, `entry header ${foreign.join(', ')}`); + if (manifest.extends.handlers && definition.handlers.length === 0) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: extends.handlers is true but ${manifest.entry} declares no .on() handlers.`); + } + if (!manifest.extends.handlers && definition.handlers.length > 0) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: ${manifest.entry} declares handlers but extends.handlers is false.`); + } + definition.handlers.forEach((handler, index) => assertDeclaredSubscription(manifest.name, manifest, handler, index)); + const hooks = imported.hooks; + const exported = Object.keys(hooks).sort(); + const declared = [...manifest.extends.hooks].sort(); + if (exported.join('\0') !== declared.join('\0')) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: extends.hooks [${manifest.extends.hooks.join(', ')}] does not match exported hooks [${exported.join(', ')}].`); + } + const baseHooks = base.definition.header.hooks ?? []; + for (const hook of manifest.extends.hooks) { + if (!baseHooks.includes(hook)) { + throw new PluginError('plugin_incompatible', `${manifest.name}: hook ${hook} is not declared by the base flow.`); + } + } + return Object.freeze({ + name: manifest.name, version: manifest.version, ref, digest: lock.digest, directory, entryPath, manifest, + handle: imported.handle, getDefinition: imported.getDefinition, handlers: Object.freeze([...definition.handlers]), + hooks, + }); +} + +export function parseHooksExport(module: Record, name: string): Readonly> { + const exported = module['hooks']; + if (exported === undefined) return Object.freeze({}); + if (typeof exported !== 'object' || exported === null || Array.isArray(exported)) { + throw new PluginError('plugin_manifest_invalid', `${name}: hooks export must be a record of functions.`); + } + const hooks: Record = {}; + for (const [key, value] of Object.entries(exported)) { + if (typeof value !== 'function') { + throw new PluginError('plugin_manifest_invalid', `${name}: hooks.${key} is not a function.`); + } + hooks[key] = value as FlowHook; + } + return Object.freeze(hooks); +} + +/** Extensions declared by the project that owns `flowPath`, verified and loaded in lock order; empty when none are declared. */ +export async function loadFlowExtensions( + flowPath: string, + base: { readonly definition: AuthoredFlowDefinition; readonly surfaceAuthority: Authority }, + options: LoadFlowExtensionsOptions, +): Promise { + const root = findPluginProject(dirname(flowPath)); + if (root === undefined) return Object.freeze([]); + const declared = reconcileDeclaredExtensions(root); + const loaded: LoadedFlowExtension[] = []; + for (const { ref, entry } of declared) loaded.push(await loadOne(root, entry, ref, base, options)); + return Object.freeze(loaded); +} + +/** The base definition with extension handlers appended in lock order; the base's own fields are untouched. */ +export function composeDefinition(base: AuthoredFlowDefinition, extensions: readonly LoadedFlowExtension[]): AuthoredFlowDefinition { + if (extensions.length === 0) return base; + return Object.freeze({ + ...base, + handlers: Object.freeze([...base.handlers, ...extensions.flatMap(extension => extension.handlers)]), + }); +} diff --git a/packages/sdk/src/flow-extension-manifest.ts b/packages/sdk/src/flow-extension-manifest.ts new file mode 100644 index 000000000..f18a8a579 --- /dev/null +++ b/packages/sdk/src/flow-extension-manifest.ts @@ -0,0 +1,190 @@ +import { Ajv } from 'ajv'; +import { providerEventTypes } from '@relayflows/surface'; +import { FLOW_HARNESSES } from './flow-requirements.js'; +import { snapshotJsonValue } from './json-value.js'; +import { PluginError, pluginKindOf } from './plugin-manifest.js'; +import { safePath } from './bundle.js'; +import { SHA, parsePluginSource, type PluginSourceInput } from './plugin-source.js'; +import { isVersionRange, parseVersion } from './semver-range.js'; + +/** + * Schema 2, `kind: "flow-extension"`: a plugin whose `entry` default-exports + * `flow()` and contributes handlers, hooks, and gates to a base flow. It is the + * same `flows-plugin.json` file and the same preflight covenant as a helper + * plugin (RFC-0001 decision 13); only the kind decides which validator reads it. + * Helper manifests (`kind` absent) never reach this module. + * + * This slice validates and records the declaration. Runtime composition is + * refused with `plugin_unsupported` (plugin-loader.ts) until the handlers slice. + */ +export interface FlowExtensionTrigger { readonly provider: string; readonly event: string; readonly actions: readonly string[] } +export interface FlowExtensionCompat { + readonly surface: string; + readonly sdk: string; + readonly base: readonly { readonly name: string; readonly version: string }[]; +} +export interface FlowExtensionPermissions { + readonly integrations: readonly string[]; + readonly harnesses: readonly string[]; + readonly mcp: readonly string[]; + /** Declared effect classes, shown for review; not enforced by this runtime (gate 8 / #442). */ + readonly writes: readonly string[]; + readonly budget?: { readonly dollars?: number; readonly wallclock?: string }; +} +export interface FlowExtensionManifest { + readonly schema: 2; + readonly kind: 'flow-extension'; + readonly name: string; + readonly version: string; + readonly description?: string; + /** Authoring copies may omit it; `flows add` records the resolved origin in the lockfile regardless. */ + readonly source?: PluginSourceInput & { readonly sha?: string }; + readonly compat: FlowExtensionCompat; + readonly entry: string; + readonly extends: { readonly handlers: boolean; readonly hooks: readonly string[] }; + readonly triggers: readonly FlowExtensionTrigger[]; + readonly permissions: FlowExtensionPermissions; + readonly preflight: { readonly credentials: readonly string[]; readonly servers: readonly string[] }; + readonly config?: Record; +} + +const TOP_LEVEL = new Set(['schema', 'kind', 'name', 'version', 'description', 'source', 'compat', 'entry', 'extends', 'triggers', 'gates', 'verbs', 'permissions', 'preflight', 'config']); +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; +const PROVIDER = /^[a-z0-9][a-z0-9-]{0,63}$/; +const WRITE_CLASS = /^[a-z0-9-]+(?::[a-z0-9_-]+)+$/; +const WALLCLOCK = /^\d+(?:ms|s|m|h|d)$/; +const MAX_DESCRIPTION = 500; +const MAX_LIST = 64; + +const object = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const invalid = (message: string): never => { throw new PluginError('plugin_manifest_invalid', message); }; + +function stringList(value: unknown, what: string, pattern: RegExp): readonly string[] { + if (!Array.isArray(value) || value.length > MAX_LIST) return invalid(`${what} must be a list of at most ${MAX_LIST} strings.`); + const out: string[] = []; + for (const entry of value) { + if (typeof entry !== 'string' || !pattern.test(entry)) return invalid(`${what} has an invalid entry ${JSON.stringify(entry)}.`); + if (out.includes(entry)) return invalid(`${what} lists ${entry} twice.`); + out.push(entry); + } + return Object.freeze(out); +} + +function compat(value: unknown): FlowExtensionCompat { + if (!object(value) || Object.keys(value).some(k => !['surface', 'sdk', 'base'].includes(k))) return invalid('compat expects surface, sdk, and base.'); + for (const key of ['surface', 'sdk'] as const) { + if (typeof value[key] !== 'string' || !isVersionRange(value[key])) return invalid(`compat.${key} must be a version range (*, x.y.z, ^x.y.z, ~x.y.z, >=x.y.z [ MAX_LIST) return invalid('compat.base must name at least one base flow.'); + const base = value.base.map(entry => { + if (!object(entry) || Object.keys(entry).some(k => !['name', 'version'].includes(k)) + || typeof entry.name !== 'string' || entry.name.trim().length === 0 || entry.name.length > 100 + || typeof entry.version !== 'string' || !isVersionRange(entry.version)) return invalid('compat.base entries are { name, version range }.'); + return Object.freeze({ name: entry.name, version: entry.version }); + }); + if (new Set(base.map(b => b.name)).size !== base.length) return invalid('compat.base names a base flow twice.'); + return Object.freeze({ surface: value.surface as string, sdk: value.sdk as string, base: Object.freeze(base) }); +} + +function triggers(value: unknown): readonly FlowExtensionTrigger[] { + if (!Array.isArray(value) || value.length > MAX_LIST) return invalid('triggers must be a list.'); + const registry = providerEventTypes as Readonly>; + const seen = new Set(); + return Object.freeze(value.map(entry => { + if (!object(entry) || Object.keys(entry).some(k => !['provider', 'event', 'actions'].includes(k)) + || typeof entry.provider !== 'string' || !PROVIDER.test(entry.provider) + || typeof entry.event !== 'string' || !IDENTIFIER.test(entry.event)) return invalid('triggers entries are { provider, event, actions }.'); + const actions = stringList(entry.actions, `triggers ${entry.provider}.${entry.event} actions`, IDENTIFIER); + const known = registry[entry.provider]; + // Fail where ingress would: an event the surface registry cannot lower is + // refused now, not after deployment on the first real delivery. + if (known === undefined) throw new PluginError('plugin_event_unroutable', `Trigger provider ${entry.provider} is not in the surface event registry.`); + const unroutable = (actions.length === 0 ? [entry.event] : actions.map(a => `${entry.event}.${a}`)).filter(type => !known.includes(type)); + if (unroutable.length > 0) throw new PluginError('plugin_event_unroutable', `Trigger ${entry.provider} ${unroutable.join(', ')} is not in the surface event registry.`); + const key = `${entry.provider}:${entry.event}`; + if (seen.has(key)) return invalid(`Trigger ${key} is declared twice.`); + seen.add(key); + return Object.freeze({ provider: entry.provider, event: entry.event, actions }); + })); +} + +function permissions(value: unknown): FlowExtensionPermissions { + if (!object(value) || Object.keys(value).some(k => !['integrations', 'harnesses', 'mcp', 'writes', 'budget'].includes(k))) { + return invalid('permissions expects integrations, harnesses, mcp, writes, and optional budget.'); + } + const harnesses = stringList(value.harnesses, 'permissions.harnesses', /^[a-z]+$/); + for (const harness of harnesses) if (!(FLOW_HARNESSES as readonly string[]).includes(harness)) return invalid(`permissions.harnesses: unknown harness ${harness}.`); + let budget: FlowExtensionPermissions['budget']; + if (value.budget !== undefined) { + if (!object(value.budget) || Object.keys(value.budget).some(k => !['dollars', 'wallclock'].includes(k))) return invalid('permissions.budget expects dollars and/or wallclock.'); + if (value.budget.dollars !== undefined && (typeof value.budget.dollars !== 'number' || !(value.budget.dollars > 0) || !Number.isFinite(value.budget.dollars))) return invalid('permissions.budget.dollars must be a positive number.'); + if (value.budget.wallclock !== undefined && (typeof value.budget.wallclock !== 'string' || !WALLCLOCK.test(value.budget.wallclock))) return invalid('permissions.budget.wallclock must be a duration such as 45m.'); + budget = Object.freeze({ ...(value.budget.dollars === undefined ? {} : { dollars: value.budget.dollars }), ...(value.budget.wallclock === undefined ? {} : { wallclock: value.budget.wallclock }) }); + } + return Object.freeze({ + integrations: stringList(value.integrations, 'permissions.integrations', PROVIDER), + harnesses, + mcp: stringList(value.mcp, 'permissions.mcp', /^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$/), + writes: stringList(value.writes, 'permissions.writes', WRITE_CLASS), + ...(budget === undefined ? {} : { budget }), + }); +} + +function source(value: unknown): FlowExtensionManifest['source'] { + if (!object(value) || Object.keys(value).some(k => !['host', 'owner', 'repo', 'sha', 'path'].includes(k)) + || value.host !== 'github' || typeof value.owner !== 'string' || typeof value.repo !== 'string' + || (value.path !== undefined && typeof value.path !== 'string') + || (value.sha !== undefined && (typeof value.sha !== 'string' || !SHA.test(value.sha)))) { + return invalid('source expects { host: "github", owner, repo, path, sha? }.'); + } + const path = (value.path as string | undefined) ?? ''; + const parsed = parsePluginSource(`github:${value.owner}/${value.repo}@${(value.sha as string | undefined) ?? 'HEAD'}${path === '' ? '' : `#${path}`}`); + return Object.freeze({ ...parsed, ...(value.sha === undefined ? {} : { sha: value.sha as string }) }); +} + +export function validateFlowExtensionManifest(input: unknown): FlowExtensionManifest { + let v: unknown; + try { v = snapshotJsonValue(input, 'plugin manifest'); } + catch { return invalid('Plugin manifest must be JSON data.'); } + if (!object(v)) return invalid('Expected a plugin manifest object.'); + if (pluginKindOf(v) !== 'flow-extension') throw new PluginError('plugin_kind_invalid', 'Expected kind "flow-extension".'); + if (v.schema !== 2) return invalid('A flow-extension manifest is schema 2.'); + const unknown = Object.keys(v).filter(k => !TOP_LEVEL.has(k)); + if (unknown.length > 0) return invalid(`Unknown manifest fields: ${unknown.join(', ')}.`); + if (!Object.hasOwn(v, 'preflight')) throw new PluginError('plugin_preflight_missing', 'Plugin must declare preflight.'); + if (typeof v.name !== 'string' || !NAME.test(v.name) || v.name.startsWith('helper-')) return invalid('name must be lowercase kebab-case and must not start with helper-.'); + if (typeof v.version !== 'string' || parseVersion(v.version) === undefined) return invalid('version must be semver x.y.z.'); + if (v.description !== undefined && (typeof v.description !== 'string' || v.description.length > MAX_DESCRIPTION)) return invalid(`description must be a string of at most ${MAX_DESCRIPTION} characters.`); + if (typeof v.entry !== 'string' || !v.entry.endsWith('.flow.ts') || !safePath(v.entry)) return invalid('entry must be a plugin-relative .flow.ts path.'); + if (!object(v.extends) || Object.keys(v.extends).some(k => !['handlers', 'hooks', 'verbs', 'gates'].includes(k)) || typeof v.extends.handlers !== 'boolean') return invalid('extends expects { handlers: boolean, hooks: [] }.'); + const hooks = stringList(v.extends.hooks ?? [], 'extends.hooks', NAME); + for (const [field, at] of [['verbs', v.extends.verbs], ['gates', v.extends.gates], ['verbs', v.verbs], ['gates', v.gates]] as const) { + if (at !== undefined && (!Array.isArray(at) || at.length > 0)) return invalid(`A flow extension declares no ${field}; ship a helper plugin beside it.`); + } + if (!v.extends.handlers && hooks.length === 0) return invalid('A flow extension must contribute handlers or at least one hook.'); + if (!object(v.preflight) || Object.keys(v.preflight).some(k => !['credentials', 'servers'].includes(k))) return invalid('Preflight requires credentials and servers arrays.'); + const credentials = stringList(v.preflight.credentials, 'preflight.credentials', IDENTIFIER); + const servers = stringList(v.preflight.servers, 'preflight.servers', /^https?:\/\/\S+$/); + for (const server of servers) { + try { if (!['http:', 'https:'].includes(new URL(server).protocol)) return invalid('Servers must be HTTP(S) URLs.'); } + catch { return invalid('Servers must be HTTP(S) URLs.'); } + } + let config: Record | undefined; + if (v.config !== undefined) { + if (!object(v.config)) return invalid('config must be a JSON Schema object.'); + try { new Ajv({ strict: false }).compile(v.config); } catch { return invalid('config is not a valid JSON Schema.'); } + config = v.config; + } + return Object.freeze({ + schema: 2, kind: 'flow-extension', name: v.name, version: v.version, + ...(v.description === undefined ? {} : { description: v.description }), + ...(v.source === undefined ? {} : { source: source(v.source) }), + compat: compat(v.compat), entry: v.entry, + extends: Object.freeze({ handlers: v.extends.handlers, hooks }), + triggers: triggers(v.triggers ?? []), + permissions: permissions(v.permissions), + preflight: Object.freeze({ credentials, servers }), + ...(config === undefined ? {} : { config }), + }); +} diff --git a/packages/sdk/src/flow-extension-submit.ts b/packages/sdk/src/flow-extension-submit.ts new file mode 100644 index 000000000..f066b3af4 --- /dev/null +++ b/packages/sdk/src/flow-extension-submit.ts @@ -0,0 +1,120 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { sha256 } from './bundle.js'; +import { CloudFlowError } from './cloud-http.js'; +import { extensionManifestOf } from './cli/add-extension.js'; +import { assertCompatible, runtimeVersions } from './flow-extension-compat.js'; +import type { LoadedAuthoredFlow } from './authored-flow-loader.js'; +import type { FlowExtensionManifest } from './flow-extension-manifest.js'; +import { probeFlowExtension, type LoadedFlowExtension } from './flow-extension-loader.js'; +import { fetchGithubPlugin, MAX_PLUGIN_TOTAL_BYTES, resolveGithubSha, type FetchLike } from './plugin-github.js'; +import { PluginError } from './plugin-manifest.js'; +import { canonicalPluginRef, parsePluginSource } from './plugin-source.js'; +import { readStoredPluginFiles } from './plugin-store.js'; + +export const MAX_EXTENSIONS_BYTES = MAX_PLUGIN_TOTAL_BYTES; + +export interface FlowExtensionFileSubmission { + readonly path: string; + readonly sha256: string; + readonly bytes: number; + readonly encoding: 'utf8' | 'base64'; + readonly content: string; +} + +export interface FlowExtensionSubmission { + readonly name: string; + readonly version: string; + readonly ref: string; + readonly digest: string; + readonly manifestSha256: string; + readonly manifest: FlowExtensionManifest; + readonly files: readonly FlowExtensionFileSubmission[]; +} + +function encodeFile(path: string, data: Uint8Array): FlowExtensionFileSubmission { + const text = Buffer.from(data).toString('utf8'); + const utf8 = Buffer.from(text, 'utf8').equals(Buffer.from(data)); + return { + path, + sha256: sha256(data), + bytes: data.length, + encoding: utf8 ? 'utf8' : 'base64', + content: utf8 ? text : Buffer.from(data).toString('base64'), + }; +} + +async function submissionFromStore(extension: LoadedFlowExtension): Promise { + const stored = await readStoredPluginFiles(extension.directory, extension.digest); + const files = stored.filter(file => file.path !== 'manifest.json').map(file => encodeFile(file.path, file.data)); + const manifestBytes = readFileSync(join(extension.directory, 'flows-plugin.json')); + return { + name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, + manifestSha256: sha256(manifestBytes), + manifest: extension.manifest, + files, + }; +} + +/** Resolve a GitHub plugin reference without writing the working tree. */ +export async function resolveExtensionSubmission( + input: string, + options: { fetch?: FetchLike; versions?: { sdk: string; surface: string } } = {}, +): Promise { + try { + const source = await resolveGithubSha(parsePluginSource(input), options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const files = fetched.files.map(file => encodeFile(file.path, file.data)); + return { + name: manifest.name, version: manifest.version, ref: canonicalPluginRef(source), + digest: fetched.digest, manifestSha256, manifest, files, + }; + } catch (error) { + if (error instanceof CloudFlowError) throw error; + if (error instanceof PluginError) throw new CloudFlowError('invalid_input', error.message); + throw new CloudFlowError('invalid_input', error instanceof Error ? error.message : String(error)); + } +} + +/** + * Installed extensions plus optional send-only `--plugin` refs. Caps the + * extensions field at 2 MB separately from the 256 KB source cap. + */ +export async function collectExtensionSubmissions( + loaded: LoadedAuthoredFlow, + extraRefs: readonly string[] = [], + options: { fetch?: FetchLike } = {}, +): Promise { + const submissions: FlowExtensionSubmission[] = []; + for (const extension of loaded.extensions) submissions.push(await submissionFromStore(extension)); + const names = new Set(submissions.map(s => s.name)); + for (const ref of extraRefs) { + const extra = await resolveExtensionSubmission(ref, options); + if (names.has(extra.name)) { + throw new CloudFlowError('invalid_input', `Plugin ${extra.name} is already in this project; omit --plugin or remove the installed copy.`); + } + names.add(extra.name); + submissions.push(extra); + } + const total = Buffer.byteLength(JSON.stringify(submissions), 'utf8'); + if (total > MAX_EXTENSIONS_BYTES) { + throw new CloudFlowError('invalid_input', `Flow extensions exceed Cloud's ${MAX_EXTENSIONS_BYTES}-byte extensions cap.`); + } + try { + for (const submission of submissions) await probeFlowExtension(submission.manifest); + } catch (error) { + if (error instanceof PluginError) throw new CloudFlowError('invalid_input', error.message); + throw error; + } + return submissions; +} + +/** Graph nodes besides the root must be the composed extensions, not `use:` children. */ +export function assertNoUseDependencies(loaded: LoadedAuthoredFlow): void { + if (loaded.graph.length !== 1 + loaded.extensions.length) { + throw new CloudFlowError('unsupported_source', + 'Cloud deploys one self-contained .flow.ts source without use dependencies.'); + } +} diff --git a/packages/sdk/src/flow-requirements.ts b/packages/sdk/src/flow-requirements.ts index e8edbc82d..7c589e590 100644 --- a/packages/sdk/src/flow-requirements.ts +++ b/packages/sdk/src/flow-requirements.ts @@ -26,9 +26,9 @@ export type FlowHarness = (typeof FLOW_HARNESSES)[number]; export interface FlowIntegrationRequirement { /** Cloud integration provider id (`slack`, `github`, `linear`, …). */ provider: string; - /** `tools`: a header declaration; `source`: a trigger or deploy target; `helper`: body use without a flag, or a YAML helper step. */ - from: 'tools' | 'source' | 'helper' | 'human'; - /** The declaration that requires it, as a reader would name it: `tools.slack`, `--on github`, `f.slack`, `f.human to`. */ + /** Where this need was declared: a header/source/helper/human route, or an extension manifest. */ + from: 'tools' | 'source' | 'helper' | 'human' | 'extension'; + /** The declaration that requires it, as a reader would name it: `tools.slack`, `--on github`, `f.slack`, or `plugin "name"`. */ detail: string; } @@ -56,6 +56,47 @@ export interface FlowRequirementsContext { projectCli?: string; } +export interface FlowExtensionRequirements { + readonly name: string; + readonly permissions: { + readonly integrations: readonly string[]; + readonly harnesses: readonly string[]; + readonly mcp: readonly string[]; + }; +} + +/** Add manifest-only needs without replacing the base flow's first declaration. */ +export function mergeFlowExtensionRequirements( + base: FlowRequirements, + extensions: readonly FlowExtensionRequirements[], +): FlowRequirements { + const integrations = [...base.integrations]; + const integrationNames = new Set(integrations.map(entry => entry.provider)); + const harnessUses = [...base.harnessUses]; + const harnessNames = new Set(base.harnesses); + const mcp = [...base.mcp]; + const mcpNames = new Set(mcp); + + for (const extension of extensions) { + for (const provider of extension.permissions.integrations) { + if (integrationNames.has(provider)) continue; + integrationNames.add(provider); + integrations.push({ provider, from: 'extension', detail: `plugin "${extension.name}"` }); + } + for (const harness of extension.permissions.harnesses) { + if (harnessNames.has(harness)) continue; + harnessNames.add(harness); + harnessUses.push({ harness: harness as FlowHarness, detail: `plugin "${extension.name}"` }); + } + for (const server of extension.permissions.mcp) { + if (mcpNames.has(server)) continue; + mcpNames.add(server); + mcp.push(server); + } + } + return { integrations, harnesses: harnessUses.map(use => use.harness), harnessUses, mcp }; +} + /** The inert subset of an authored definition this module reads. */ export interface RequirementsFlowDefinition { readonly header?: { readonly tools?: Readonly> }; diff --git a/packages/sdk/src/plugin-github.ts b/packages/sdk/src/plugin-github.ts new file mode 100644 index 000000000..170eabb0a --- /dev/null +++ b/packages/sdk/src/plugin-github.ts @@ -0,0 +1,98 @@ +import { payloadManifest, safePath, sha256 } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; +import { SHA, type PluginSourceInput, type PluginSourceRef } from './plugin-source.js'; + +/** + * Public, unauthenticated GitHub reads for flow-extension plugins — the same + * posture as Cloud's deploy links: no credential ever travels to GitHub, so a + * private repository simply answers 404 and is reported as unresolved. + * + * Two calls resolve a ref and enumerate a tree; blobs come from + * raw.githubusercontent.com pinned to the resolved commit. Every byte count is + * checked against the tree listing, and the tree is refused if GitHub + * truncated it, if it contains a symlink or submodule under the plugin path, + * or if any file or the whole plugin exceeds the bounds below. + */ +export type FetchLike = (url: string, init: { headers: Record; signal: AbortSignal }) => Promise; + +export const MAX_PLUGIN_FILE_BYTES = 256_000; +export const MAX_PLUGIN_TOTAL_BYTES = 2_000_000; +export const MAX_PLUGIN_FILES = 500; +const TIMEOUT_MS = 15_000; +const API = 'https://api.github.com'; +const RAW = 'https://raw.githubusercontent.com'; + +export interface FetchedPluginFile { readonly path: string; readonly data: Buffer } +export interface FetchedPlugin { + readonly source: PluginSourceRef; + /** Plugin-relative paths, sorted. */ + readonly files: readonly FetchedPluginFile[]; + /** sha256 of `payloadManifest(files)`: the content identity persisted as `@sha256:`. */ + readonly digest: string; +} + +async function request(fetch: FetchLike, url: string, accept: string): Promise { + let response: Response; + try { + response = await fetch(url, { headers: { accept, 'user-agent': 'relayflows-sdk' }, signal: AbortSignal.timeout(TIMEOUT_MS) }); + } catch (error) { + throw new PluginError('plugin_fetch_failed', `GitHub request failed: ${url} (${error instanceof Error ? error.message : String(error)}).`); + } + if (response.status === 404) throw new PluginError('plugin_source_unresolved', `Not found on GitHub (or not public): ${url}.`); + if (!response.ok) throw new PluginError('plugin_fetch_failed', `GitHub answered ${response.status} for ${url}.`); + return response; +} + +/** Branch, tag, or commit → the commit sha it names right now; a sha input is verified to exist. */ +export async function resolveGithubSha(source: PluginSourceInput, fetch: FetchLike = globalThis.fetch): Promise { + const url = `${API}/repos/${source.owner}/${source.repo}/commits/${encodeURIComponent(source.ref)}`; + const sha = (await (await request(fetch, url, 'application/vnd.github.sha')).text()).trim(); + if (!SHA.test(sha)) throw new PluginError('plugin_fetch_failed', `GitHub returned a malformed commit sha for ${source.ref}.`); + if (SHA.test(source.ref) && sha !== source.ref) throw new PluginError('plugin_source_unresolved', `Commit ${source.ref} resolved to ${sha}.`); + return Object.freeze({ ...source, sha }); +} + +interface TreeEntry { path: string; mode: string; type: string; size?: number } + +async function listTree(source: PluginSourceRef, fetch: FetchLike): Promise { + const url = `${API}/repos/${source.owner}/${source.repo}/git/trees/${source.sha}?recursive=1`; + let body: unknown; + try { body = await (await request(fetch, url, 'application/vnd.github+json')).json(); } + catch { throw new PluginError('plugin_fetch_failed', 'GitHub tree listing is not JSON.'); } + const tree = typeof body === 'object' && body !== null ? (body as { tree?: unknown; truncated?: unknown }) : {}; + if (tree.truncated === true) throw new PluginError('plugin_fetch_failed', 'GitHub truncated the tree listing; the repository is too large to enumerate safely.'); + if (!Array.isArray(tree.tree)) throw new PluginError('plugin_fetch_failed', 'GitHub tree listing has no entries.'); + return tree.tree.filter((e): e is TreeEntry => typeof e === 'object' && e !== null + && typeof (e as TreeEntry).path === 'string' && typeof (e as TreeEntry).mode === 'string' && typeof (e as TreeEntry).type === 'string'); +} + +/** Enumerate and download the plugin directory at the pinned commit, bounded and verified. */ +export async function fetchGithubPlugin(source: PluginSourceRef, fetch: FetchLike = globalThis.fetch): Promise { + const prefix = source.path === '' ? '' : `${source.path}/`; + const entries = (await listTree(source, fetch)).filter(e => e.path.startsWith(prefix)); + if (source.path !== '' && entries.length === 0) throw new PluginError('plugin_source_unresolved', `${source.path} does not exist at ${source.sha}.`); + const blobs: { path: string; size: number }[] = []; + let total = 0; + for (const entry of entries) { + const relative = entry.path.slice(prefix.length); + if (entry.type === 'tree') continue; + if (entry.type === 'commit') throw new PluginError('plugin_path_invalid', `${entry.path} is a submodule; plugins must be plain files.`); + if (entry.mode === '120000') throw new PluginError('plugin_path_invalid', `${entry.path} is a symlink; plugins must be plain files.`); + if (entry.type !== 'blob' || !safePath(relative)) throw new PluginError('plugin_path_invalid', `${entry.path} is not a plain repository file.`); + if (!Number.isSafeInteger(entry.size) || entry.size! < 0) throw new PluginError('plugin_fetch_failed', `${entry.path} has no size in the tree listing.`); + if (entry.size! > MAX_PLUGIN_FILE_BYTES) throw new PluginError('plugin_too_large', `${entry.path} is ${entry.size} bytes; the limit is ${MAX_PLUGIN_FILE_BYTES}.`); + total += entry.size!; + if (total > MAX_PLUGIN_TOTAL_BYTES) throw new PluginError('plugin_too_large', `Plugin exceeds ${MAX_PLUGIN_TOTAL_BYTES} bytes in total.`); + blobs.push({ path: relative, size: entry.size! }); + if (blobs.length > MAX_PLUGIN_FILES) throw new PluginError('plugin_too_large', `Plugin has more than ${MAX_PLUGIN_FILES} files.`); + } + if (!blobs.some(b => b.path === 'flows-plugin.json')) throw new PluginError('plugin_manifest_missing', `${source.owner}/${source.repo}@${source.sha}${prefix === '' ? '' : `#${source.path}`} has no flows-plugin.json.`); + const files: FetchedPluginFile[] = []; + for (const blob of blobs.sort((a, b) => a.path < b.path ? -1 : 1)) { + const url = `${RAW}/${source.owner}/${source.repo}/${source.sha}/${prefix}${blob.path}`; + const data = Buffer.from(await (await request(fetch, url, 'application/octet-stream')).arrayBuffer()); + if (data.length !== blob.size) throw new PluginError('plugin_source_drift', `${blob.path}: fetched ${data.length} bytes, tree lists ${blob.size}.`); + files.push(Object.freeze({ path: blob.path, data })); + } + return Object.freeze({ source, files: Object.freeze(files), digest: sha256(payloadManifest(files)) }); +} diff --git a/packages/sdk/src/plugin-loader.ts b/packages/sdk/src/plugin-loader.ts index 8911590ea..797a89e6a 100644 --- a/packages/sdk/src/plugin-loader.ts +++ b/packages/sdk/src/plugin-loader.ts @@ -5,6 +5,7 @@ import { Ajv } from 'ajv'; import type { Step } from '@relayflows/surface'; import { snapshotJsonValue } from './json-value.js'; import { assertSupportedPlugin, PluginError, pluginPackageName, validatePluginManifest, type PluginManifest, type PluginVerb } from './plugin-manifest.js'; +import { isGithubPluginRef } from './plugin-source.js'; export interface LoadedPlugin { readonly directory: string; readonly manifest: PluginManifest } export function findPluginProject(start: string): string | undefined { @@ -47,8 +48,11 @@ export async function loadPlugins(start: string): Promise typeof p === 'string')))) { throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be package names.'); } + // Flow extensions (github:… entries) are not helpers: the authored flow + // loader verifies and composes them (flow-extension-loader.ts). Here they + // are simply not helper packages, so they are left out of the helper set. const scope = join(root, 'node_modules/@flows'); - const names = new Set((config.plugins as string[] | undefined)?.map(pluginPackageName)); + const names = new Set((config.plugins as string[] | undefined)?.filter(p => !isGithubPluginRef(p)).map(pluginPackageName)); if (existsSync(scope)) for (const name of readdirSync(scope).sort()) { if (name.startsWith('helper-') && !names.has(`@flows/${name}`)) { throw new PluginError('plugin_unlisted', `@flows/${name} is installed but not declared in flows.json plugins. Run flows add ${name}.`); diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts new file mode 100644 index 000000000..19fc90495 --- /dev/null +++ b/packages/sdk/src/plugin-lock.ts @@ -0,0 +1,194 @@ +import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PluginError } from './plugin-manifest.js'; +import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; + +/** + * `flows.lock.json` — the project's plugin provenance. `flows.json.plugins` + * says *what* is declared; the lockfile says exactly which bytes that meant: + * the commit, the content digest, the manifest hash, and the order the + * operator declared. Version 2 is also the sealed bundle `lockfile.json` + * shape when the bundle carries plugins (RFC-0001 decision 14). + */ +export const PLUGIN_LOCK_FILE = 'flows.lock.json'; +export const PLUGIN_LOCK_VERSION = 2; +const FLOWS_FILE = 'flows.json'; + +export interface PluginLockEntry { + readonly name: string; + readonly kind: 'flow-extension'; + readonly version: string; + readonly source: { readonly host: 'github'; readonly owner: string; readonly repo: string; readonly sha: string; readonly path: string }; + /** sha256 of the payload manifest — the `@sha256:` in `.flows/plugins`. */ + readonly digest: string; + /** sha256 of the `flows-plugin.json` bytes as installed. */ + readonly manifestSha256: string; + /** + * 1-based position among the flow-extension (`github:`) entries of + * `flows.json.plugins`, in declaration order. Helper entries interspersed in + * that list do not count, so the order is the composition order exactly. + */ + readonly order: number; + readonly resolvedAt: string; +} +export interface PluginLock { readonly version: 2; readonly plugins: readonly PluginLockEntry[] } + +const HEX64 = /^[0-9a-f]{64}$/; +const object = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const invalid = (message: string): never => { throw new PluginError('plugin_lock_invalid', `${PLUGIN_LOCK_FILE}: ${message}`); }; + +export function parsePluginLock(input: unknown): PluginLock { + if (!object(input) || input.version !== PLUGIN_LOCK_VERSION || !Array.isArray(input.plugins) + || Object.keys(input).some(k => !['version', 'plugins'].includes(k))) return invalid(`expected { version: ${PLUGIN_LOCK_VERSION}, plugins: [] }.`); + const names = new Set(); + const plugins = input.plugins.map((entry, index) => { + if (!object(entry) || Object.keys(entry).sort().join(',') !== 'digest,kind,manifestSha256,name,order,resolvedAt,source,version' + || entry.kind !== 'flow-extension' || typeof entry.name !== 'string' || typeof entry.version !== 'string' + || typeof entry.digest !== 'string' || !HEX64.test(entry.digest) + || typeof entry.manifestSha256 !== 'string' || !HEX64.test(entry.manifestSha256) + || entry.order !== index + 1 || typeof entry.resolvedAt !== 'string' || Number.isNaN(Date.parse(entry.resolvedAt)) + || !object(entry.source) || entry.source.host !== 'github' || typeof entry.source.owner !== 'string' + || typeof entry.source.repo !== 'string' || typeof entry.source.sha !== 'string' || !SHA.test(entry.source.sha) + || typeof entry.source.path !== 'string') return invalid(`plugins[${index}] is malformed.`); + if (names.has(entry.name)) return invalid(`plugin ${entry.name} is listed twice.`); + names.add(entry.name); + const source = parseCanonicalPluginRef(canonicalPluginRef({ host: 'github', owner: entry.source.owner, repo: entry.source.repo, ref: entry.source.sha, sha: entry.source.sha, path: entry.source.path })); + return Object.freeze({ + name: entry.name, kind: 'flow-extension' as const, version: entry.version, + source: Object.freeze({ host: 'github' as const, owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }), + digest: entry.digest, manifestSha256: entry.manifestSha256, order: entry.order, resolvedAt: entry.resolvedAt, + }); + }); + return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze(plugins) }); +} + +/** Absent file → empty lock; unreadable or malformed → refusal. */ +export function readPluginLock(root: string): PluginLock { + recoverFlowsAndLock(root); + const path = join(root, PLUGIN_LOCK_FILE); + if (!existsSync(path)) return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze([]) }); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(path, 'utf8')); } + catch { return invalid('not valid JSON.'); } + return parsePluginLock(parsed); +} + +export function writePluginLock(root: string, lock: PluginLock): void { + writeFileSync(join(root, PLUGIN_LOCK_FILE), `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); +} + +/** + * Finish or abort a two-file update left by a process crash. The lock temp is + * written first: by itself it is only preparation and can be discarded. Once + * the config temp exists, both complete snapshots exist and recovery rolls + * them forward in the same lock-then-declaration order as the writer. + */ +export function recoverFlowsAndLock(root: string, configPath = join(root, FLOWS_FILE)): void { + const jsonTmp = `${configPath}.tmp`; + const lockPath = join(root, PLUGIN_LOCK_FILE); + const lockTmp = `${lockPath}.tmp`; + const hasJson = existsSync(jsonTmp); + const hasLock = existsSync(lockTmp); + if (!hasJson && !hasLock) return; + if (!hasJson) { + unlinkSync(lockTmp); + return; + } + let pendingConfig: unknown; + try { pendingConfig = JSON.parse(readFileSync(jsonTmp, 'utf8')); } + catch { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + return invalid('pending transaction has invalid flows.json.tmp.'); + } + if (!object(pendingConfig) || !Array.isArray(pendingConfig.plugins) + || !pendingConfig.plugins.every(plugin => typeof plugin === 'string')) { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + return invalid('pending transaction has invalid flows.json.tmp.'); + } + const lockSource = hasLock ? lockTmp : lockPath; + let pendingLock: unknown; + try { pendingLock = JSON.parse(readFileSync(lockSource, 'utf8')); } + catch { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + return invalid('pending transaction has no valid lock snapshot.'); + } + try { parsePluginLock(pendingLock); } + catch (error) { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + throw error; + } + if (hasLock) renameSync(lockTmp, lockPath); + renameSync(jsonTmp, configPath); +} + +/** Write complete snapshots, then recover them as one roll-forward transaction. */ +export function writeFlowsAndLock(root: string, configPath: string, config: Record, plugins: readonly string[], lock: PluginLock): void { + const jsonTmp = `${configPath}.tmp`; + const lockPath = join(root, PLUGIN_LOCK_FILE); + const lockTmp = `${lockPath}.tmp`; + writeFileSync(lockTmp, `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); + writeFileSync(jsonTmp, `${JSON.stringify({ ...config, plugins: [...plugins] }, null, 2)}\n`); + recoverFlowsAndLock(root, configPath); +} + +/** + * Rebuild the entry list in `flows.json.plugins` order: `order` is derived from + * the declaration list, never stored independently, so the two cannot disagree. + * Lock entries that are no longer declared are dropped; a declaration with no + * matching lock entry is a refusal. + */ +export function lockForDeclared(lock: PluginLock, declared: readonly string[]): PluginLock { + const byRef = new Map(lock.plugins.map(p => [canonicalPluginRef({ ...p.source, ref: p.source.sha }), p])); + const plugins = declared.filter(isGithubPluginRef).map((ref, index) => { + const found = byRef.get(ref); + if (found === undefined) return invalid(`flows.json declares ${ref} but the lockfile has no entry for it; run flows add ${ref}.`); + return Object.freeze({ ...found, order: index + 1 }); + }); + return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze(plugins) }); +} + +export function lockWithPlugin( + lock: PluginLock, declared: readonly string[], + entry: Omit, +): PluginLock { + const byRef = new Map(lock.plugins.map(p => [canonicalPluginRef({ ...p.source, ref: p.source.sha }), p])); + byRef.set(canonicalPluginRef({ ...entry.source, ref: entry.source.sha }), { ...entry, kind: 'flow-extension', order: 0 }); + return lockForDeclared({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze([...byRef.values()]) }, declared); +} + +/** Lock entries paired with their declared reference, in declaration order. */ +export function lockedPlugins(lock: PluginLock): readonly { ref: string; entry: PluginLockEntry; source: PluginSourceRef }[] { + return lock.plugins.map(entry => { + const source = { ...entry.source, ref: entry.source.sha }; + return { ref: canonicalPluginRef(source), entry, source }; + }); +} + +/** The `github:` entries of `flows.json.plugins`, in declaration order; helper entries are left out. */ +export function declaredExtensionRefs(root: string): readonly string[] { + recoverFlowsAndLock(root); + let config: { plugins?: unknown }; + try { config = JSON.parse(readFileSync(join(root, 'flows.json'), 'utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } + if (config === null || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every(p => typeof p === 'string')))) { + throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be strings.'); + } + return Object.freeze(((config.plugins as string[] | undefined) ?? []).filter(isGithubPluginRef)); +} + +/** + * The three records that must agree before an extension is trusted: + * `flows.json.plugins` (declaration), `flows.lock.json` (provenance), and — + * checked by the caller against the returned digests — `.flows/plugins` + * (bytes). Any declaration without a lock entry, lock entry without a + * declaration, or order disagreement is `plugin_lock_invalid`. + */ +export function reconcileDeclaredExtensions(root: string): readonly { ref: string; entry: PluginLockEntry; source: PluginSourceRef }[] { + const declared = declaredExtensionRefs(root); + const locked = lockedPlugins(readPluginLock(root)); + const lockedRefs = locked.map(p => p.ref); + for (const ref of declared) if (!lockedRefs.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.json declares ${ref} but flows.lock.json has no entry for it.`); + for (const ref of lockedRefs) if (!declared.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.lock.json records ${ref} but flows.json does not declare it.`); + if (declared.some((ref, index) => lockedRefs[index] !== ref)) throw new PluginError('plugin_lock_invalid', 'flows.lock.json order differs from flows.json.plugins.'); + return locked; +} diff --git a/packages/sdk/src/plugin-manifest.ts b/packages/sdk/src/plugin-manifest.ts index ba133444a..fb4791d79 100644 --- a/packages/sdk/src/plugin-manifest.ts +++ b/packages/sdk/src/plugin-manifest.ts @@ -5,6 +5,10 @@ export const PLUGIN_FAILURE_KINDS = [ 'plugin_unknown', 'plugin_unlisted', 'plugin_install_failed', 'plugin_manifest_missing', 'plugin_manifest_invalid', 'plugin_preflight_missing', 'plugin_verb_unknown_primitive', 'plugin_unsupported', 'plugin_credential_missing', 'plugin_server_unreachable', + // schema 2 flow extensions (see flow-extension-manifest.ts, plugin-source.ts, plugin-github.ts) + 'plugin_kind_invalid', 'plugin_incompatible', 'plugin_event_unroutable', 'plugin_source_invalid', + 'plugin_source_unresolved', 'plugin_fetch_failed', 'plugin_path_invalid', 'plugin_too_large', + 'plugin_source_drift', 'plugin_lock_invalid', ] as const; export type PluginFailureKind = typeof PLUGIN_FAILURE_KINDS[number]; export class PluginError extends Error { @@ -16,6 +20,15 @@ export interface PluginVerb { lowersTo: 'run' | 'llm' | 'agent' | 'effect' | 'wait'; args: Record; } +/** The kind a `flows-plugin.json` declares; absent means the schema-1 helper plugin. */ +export type PluginKind = 'helper' | 'flow-extension'; +export function pluginKindOf(input: unknown): PluginKind { + const kind = typeof input === 'object' && input !== null && !Array.isArray(input) ? (input as { kind?: unknown }).kind : undefined; + if (kind === undefined || kind === 'helper') return 'helper'; + if (kind === 'flow-extension') return 'flow-extension'; + throw new PluginError('plugin_kind_invalid', `Unknown plugin kind ${JSON.stringify(kind)}; expected "helper" or "flow-extension".`); +} +/** A schema-1 helper plugin: verbs that lower to kernel primitives. */ export interface PluginManifest { name: string; version: string; @@ -41,6 +54,10 @@ export function validatePluginManifest(input: unknown, packageName?: string): Pl catch { throw new PluginError('plugin_manifest_invalid', 'Plugin manifest must be JSON data.'); } const invalid = (message: string): never => { throw new PluginError('plugin_manifest_invalid', message); }; if (!object(v)) return invalid('Expected a plugin manifest object.'); + if (pluginKindOf(v) !== 'helper') { + throw new PluginError('plugin_kind_invalid', 'A flow-extension manifest installs from a GitHub reference (flows add github:/@#), not as a helper package.'); + } + if (v.schema !== undefined && v.schema !== 1) return invalid('Helper plugin manifests are schema 1.'); if (!Object.hasOwn(v, 'preflight')) throw new PluginError('plugin_preflight_missing', 'Plugin must declare preflight.'); if (typeof v.name !== 'string' || typeof v.version !== 'string' || !v.version.trim()) return invalid('Plugin name and version are required.'); const resolved = pluginPackageName(v.name); diff --git a/packages/sdk/src/plugin-source.ts b/packages/sdk/src/plugin-source.ts new file mode 100644 index 000000000..7eedcea56 --- /dev/null +++ b/packages/sdk/src/plugin-source.ts @@ -0,0 +1,91 @@ +import { safePath } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; + +/** + * Where a flow-extension plugin comes from: a public GitHub repository, pinned + * to a commit. A branch or tag is accepted as *input* only; what gets written + * to `flows.json` and the lockfile is always the canonical 40-hex form, + * `github:/@#`, so a later reader can never resolve + * to different bytes than the installer saw. + */ +export interface PluginSourceInput { + readonly host: 'github'; + readonly owner: string; + readonly repo: string; + /** Branch, tag, or commit as typed. */ + readonly ref: string; + /** Directory inside the repository holding `flows-plugin.json`; `''` is the root. */ + readonly path: string; +} +export interface PluginSourceRef extends PluginSourceInput { + /** Exactly 40 lowercase hex characters. */ + readonly sha: string; +} + +const OWNER = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; +const REPO = /^[A-Za-z0-9_.-]{1,100}$/; +const REF = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$/; +export const SHA = /^[0-9a-f]{40}$/; + +function invalid(message: string): never { + throw new PluginError('plugin_source_invalid', message); +} + +export function isGithubPluginRef(value: string): boolean { + return value.startsWith('github:') || /^https:\/\/github\.com\//.test(value); +} + +/** + * Accepts `github:/@#`, + * `https://github.com///tree//` (or `/blob/`), and + * `github:/@` for a root-level plugin. + * + * In the URL form the ref is the single segment after `tree/`: GitHub itself + * disambiguates `tree/feat/x/dir` against the repository's refs, which an + * offline parser cannot. A ref containing `/` must use the `github:` form, + * where `@ref#path` is unambiguous. + */ +export function parsePluginSource(input: string): PluginSourceInput { + if (input.length > 2048) return invalid('Plugin source is too long.'); + let owner: string | undefined, repo: string | undefined, ref: string | undefined, path = ''; + if (input.startsWith('github:')) { + const m = /^github:([^/@#]+)\/([^/@#]+)@([^#]+)(?:#(.*))?$/.exec(input); + if (!m) return invalid('Expected github:/@[#].'); + [, owner, repo, ref] = m; + path = m[4] ?? ''; + } else if (/^https:\/\/github\.com\//.test(input)) { + let url: URL; + try { url = new URL(input); } catch { return invalid('Plugin source is not a valid URL.'); } + if (url.username || url.password || url.search || url.hash) return invalid('Plugin URL must not carry credentials, a query, or a fragment.'); + const m = /^\/([^/]+)\/([^/]+)\/(?:tree|blob)\/([^/]+)(?:\/(.*))?$/.exec(url.pathname); + if (!m) return invalid('Expected https://github.com///tree//.'); + try { + [, owner, repo, ref] = m.map(part => part === undefined ? part : decodeURIComponent(part)); + path = m[4] === undefined ? '' : decodeURIComponent(m[4]); + } catch { + return invalid('Plugin URL contains invalid percent-encoding.'); + } + } else return invalid('Expected a github: reference or a https://github.com/ URL.'); + if (owner === undefined || !OWNER.test(owner)) return invalid('Invalid GitHub owner.'); + if (repo === undefined || !REPO.test(repo) || repo === '.' || repo === '..') return invalid('Invalid GitHub repository name.'); + if (repo.endsWith('.git')) repo = repo.slice(0, -4); + if (ref === undefined || !REF.test(ref) || ref.includes('..') || ref.endsWith('/') || ref.endsWith('.lock')) return invalid('Invalid git ref.'); + path = path.replace(/\/+$/, ''); + if (path !== '' && !safePath(path)) return invalid('Plugin path must be repository-relative without traversal.'); + if (path.split('/').some(part => part === 'flows-plugin.json')) return invalid('Plugin path names the directory holding flows-plugin.json, not the file.'); + return Object.freeze({ host: 'github', owner, repo, ref, path }); +} + +/** The one spelling that is ever persisted. */ +export function canonicalPluginRef(source: PluginSourceRef): string { + return `github:${source.owner}/${source.repo}@${source.sha}${source.path === '' ? '' : `#${source.path}`}`; +} + +/** Parses a persisted reference; refuses anything but the canonical sha form. */ +export function parseCanonicalPluginRef(value: string): PluginSourceRef { + const parsed = parsePluginSource(value); + if (!value.startsWith('github:') || !SHA.test(parsed.ref)) { + throw new PluginError('plugin_source_invalid', `Persisted plugin reference must be github:/@[#], got ${value}.`); + } + return Object.freeze({ ...parsed, sha: parsed.ref }); +} diff --git a/packages/sdk/src/plugin-store.ts b/packages/sdk/src/plugin-store.ts new file mode 100644 index 000000000..8ff7d232f --- /dev/null +++ b/packages/sdk/src/plugin-store.ts @@ -0,0 +1,105 @@ +import { lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { payloadManifest, safePath, sha256 } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; + +/** + * Where a flow-extension plugin's bytes live inside a project: + * `/.flows/plugins/@sha256:/`, content-addressed like + * the bundle cache and never `node_modules`. `manifest.json` in that directory + * is the payload manifest whose sha256 is the digest, so a directory can be + * re-verified without the network — and, because the lockfile records the + * same digest, drift between what was installed and what is on disk is a + * refusal, not a surprise. + */ +export const PLUGIN_STORE = '.flows/plugins'; + +export function pluginStoreDirectory(root: string, name: string, digest: string): string { + return join(resolve(root), PLUGIN_STORE, `${name}@sha256:${digest}`); +} + +export interface StoredPluginFile { readonly path: string; readonly data: Uint8Array } + +/** Write the files atomically; an existing directory is verified instead of overwritten. */ +export async function materializePlugin(root: string, name: string, files: readonly StoredPluginFile[]): Promise<{ directory: string; digest: string }> { + const manifest = payloadManifest(files); + const digest = sha256(manifest); + const directory = pluginStoreDirectory(root, name, digest); + let exists = false; + try { await lstat(directory); exists = true; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + if (exists) { await verifyStoredPlugin(directory, digest); return { directory, digest }; } + const parent = dirname(directory); + await mkdir(parent, { recursive: true }); + const staging = await mkdtemp(join(parent, '.install-')); + try { + for (const file of files) { + if (!safePath(file.path) || file.path === 'manifest.json') throw new PluginError('plugin_path_invalid', `${file.path}: invalid plugin path.`); + await mkdir(dirname(join(staging, file.path)), { recursive: true }); + await writeFile(join(staging, file.path), file.data, { mode: 0o644 }); + } + await writeFile(join(staging, 'manifest.json'), manifest); + try { await rename(staging, directory); } + catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; + await verifyStoredPlugin(directory, digest); + } + } finally { await rm(staging, { recursive: true, force: true }); } + return { directory, digest }; +} + +async function regularFile(root: string, path: string): Promise { + const parts = path.split('/'); + for (let i = 1; i <= parts.length; i++) { + const stat = await lstat(join(root, ...parts.slice(0, i))); + if (i === parts.length ? !stat.isFile() : !stat.isDirectory()) throw new PluginError('plugin_source_drift', `${path}: expected a regular file, without symlinks.`); + } + return readFile(join(root, path)); +} + +/** Re-hash a materialized plugin and compare with the digest the lockfile recorded. */ +export async function verifyStoredPlugin(directory: string, expectedDigest: string): Promise { + const drift = (message: string): never => { throw new PluginError('plugin_source_drift', `${directory}: ${message}`); }; + let raw: string; + try { + if (!(await lstat(directory)).isDirectory()) return drift('not a directory'); + raw = (await regularFile(directory, 'manifest.json')).toString('utf8'); + } catch (error) { return drift(error instanceof PluginError ? error.message : 'manifest.json is missing'); } + if (sha256(raw) !== expectedDigest) return drift('manifest.json digest differs from the lockfile'); + let entries: { path: string; sha256: string; bytes: number }[]; + try { entries = JSON.parse(raw); if (!Array.isArray(entries)) throw new Error(); } + catch { return drift('manifest.json is not a manifest'); } + const paths = new Set(); + for (const entry of entries) { + if (typeof entry?.path !== 'string' || !safePath(entry.path)) return drift('manifest.json lists an invalid path'); + let data: Buffer; + try { data = await regularFile(directory, entry.path); } + catch (error) { return drift(error instanceof PluginError ? error.message : `${entry.path} is missing`); } + if (data.length !== entry.bytes || sha256(data) !== entry.sha256) return drift(`${entry.path} changed since installation`); + paths.add(entry.path); + } + await rejectExtras(directory, '', new Set([...paths, 'manifest.json']), drift); +} + +/** Re-verify, then return every stored file including the payload `manifest.json`. */ +export async function readStoredPluginFiles(directory: string, expectedDigest: string): Promise { + await verifyStoredPlugin(directory, expectedDigest); + const manifest = await regularFile(directory, 'manifest.json'); + const entries = JSON.parse(manifest.toString('utf8')) as { path: string }[]; + const files = [{ path: 'manifest.json', data: manifest }]; + for (const entry of entries) files.push({ path: entry.path, data: await regularFile(directory, entry.path) }); + return files; +} + +/** Drop a materialized plugin directory. Missing is a no-op. */ +export async function removeStoredPlugin(directory: string): Promise { + await rm(directory, { recursive: true, force: true }); +} + +async function rejectExtras(root: string, prefix: string, paths: Set, drift: (m: string) => never): Promise { + for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { + const path = prefix + entry.name; + if (entry.isDirectory() && [...paths].some(file => file.startsWith(`${path}/`))) await rejectExtras(root, `${path}/`, paths, drift); + else if (!entry.isFile() || !paths.has(path)) drift(`${path}: unlisted file or unsupported file type`); + } +} diff --git a/packages/sdk/src/semver-range.ts b/packages/sdk/src/semver-range.ts new file mode 100644 index 000000000..3a2cfc8ed --- /dev/null +++ b/packages/sdk/src/semver-range.ts @@ -0,0 +1,74 @@ +/** + * The small semver subset plugin `compat` ranges may use. Deliberately not a + * dependency on `semver`: a plugin's compatibility claim has to be checkable by + * every reader — Cloud, the CLI, a catalog — from one short, obvious rule. + * + * Accepted: `*`, `x.y.z`, `^x.y.z`, `~x.y.z`, `>=x.y.z`, `>=x.y.z =)?\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?: <\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)?)$/; + +interface Parsed { readonly triple: readonly [number, number, number]; readonly pre?: string } + +export function parseVersion(value: string): Parsed | undefined { + const m = VERSION.exec(value); + if (!m) return undefined; + const triple = [Number(m[1]), Number(m[2]), Number(m[3])] as const; + if (triple.some(n => !Number.isSafeInteger(n))) return undefined; + return m[4] === undefined ? { triple } : { triple, pre: m[4] }; +} + +export function isVersionRange(value: string): boolean { + return RANGE.test(value); +} + +function compare(a: Parsed, b: Parsed): number { + for (let i = 0; i < 3; i++) if (a.triple[i] !== b.triple[i]) return a.triple[i]! - b.triple[i]!; + if (a.pre === b.pre) return 0; + if (a.pre === undefined) return 1; + if (b.pre === undefined) return -1; + return comparePrerelease(a.pre, b.pre); +} + +function comparePrerelease(a: string, b: string): number { + const as = a.split('.'); + const bs = b.split('.'); + const n = Math.max(as.length, bs.length); + for (let i = 0; i < n; i++) { + const left = as[i]; + const right = bs[i]; + if (left === undefined) return -1; + if (right === undefined) return 1; + const leftNum = /^\d+$/.test(left) ? Number(left) : undefined; + const rightNum = /^\d+$/.test(right) ? Number(right) : undefined; + if (leftNum !== undefined && rightNum !== undefined) { + if (leftNum !== rightNum) return leftNum - rightNum; + continue; + } + if (leftNum !== undefined) return -1; + if (rightNum !== undefined) return 1; + if (left !== right) return left < right ? -1 : 1; + } + return 0; +} + +/** Whether `version` satisfies `range`; false for malformed input rather than a throw. */ +export function satisfiesRange(version: string, range: string): boolean { + const v = parseVersion(version); + if (v === undefined || !isVersionRange(range)) return false; + if (range === '*') return true; + const [lowerText, upperText] = range.split(' ') as [string, string | undefined]; + const operator = /^[\^~]|^>=/.exec(lowerText)?.[0] ?? ''; + const lower = parseVersion(lowerText.slice(operator.length))!; + if (compare(v, lower) < 0) return false; + if (operator === '') return compare(v, lower) === 0; + let upper: Parsed | undefined; + if (upperText !== undefined) upper = parseVersion(upperText.slice(1)); + else if (operator === '^') { + const [major, minor] = lower.triple; + upper = major > 0 ? { triple: [major + 1, 0, 0] } : minor > 0 ? { triple: [0, minor + 1, 0] } : { triple: [0, 0, lower.triple[2] + 1] }; + } else if (operator === '~') upper = { triple: [lower.triple[0], lower.triple[1] + 1, 0] }; + return upper === undefined || compare(v, upper) < 0; +} diff --git a/packages/sdk/tests/authored-flow.test.ts b/packages/sdk/tests/authored-flow.test.ts index ebe1fb449..32da08917 100644 --- a/packages/sdk/tests/authored-flow.test.ts +++ b/packages/sdk/tests/authored-flow.test.ts @@ -140,6 +140,12 @@ describe('authored flow journal executor', () => { async (f) => f.done('success'), ), disconnectedJournal)).rejects.toMatchObject({ code: 'unsupported_header' }); + await expect(executeAuthoredFlow(flow( + 'versioned-hooks', + { version: '2.0.22', hooks: ['merge-gate'], budget: { dollars: 1 } }, + async (f) => f.done('success'), + ), disconnectedJournal)).rejects.not.toMatchObject({ code: 'unsupported_header' }); + // Predicate .gate(fn) is accepted (its VERDICT is journaled as a lowered // `.gate` run once the step completes), so with a disconnected // journal it refuses on the run.start path like any other step. What is diff --git a/packages/sdk/tests/authored-hooks.test.ts b/packages/sdk/tests/authored-hooks.test.ts new file mode 100644 index 000000000..342bb3dbc --- /dev/null +++ b/packages/sdk/tests/authored-hooks.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import type { Ctx } from '@relayflows/surface'; +import { createHookEvaluator } from '../src/authored-hooks.js'; +import type { LoadedFlowExtension } from '../src/flow-extension-loader.js'; +import type { JournalClient } from '../src/journal-client.js'; + +function fakeJournal() { + const streams = new Map(); + const journal = { + async streamRead(_run: string, stream: string, offset: number) { + const messages = (streams.get(stream) ?? []).slice(offset); + return { messages: messages.map(message => ({ message })), next_offset: (streams.get(stream) ?? []).length }; + }, + async streamAppend(_run: string, stream: string, message: unknown) { + const list = streams.get(stream) ?? []; + list.push(message); + streams.set(stream, list); + return { offset: list.length }; + }, + } as unknown as JournalClient; + return { journal, streams }; +} + +function extension(name: string, impl: (f: Ctx, input: unknown) => Promise): LoadedFlowExtension { + return { name, hooks: { 'merge-gate': impl } } as LoadedFlowExtension; +} + +describe('hook AND composition', () => { + const ctx = {} as Ctx; + + it('is a journaled no-op returning true when nothing implements the hook', async () => { + const { journal, streams } = fakeJournal(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], extensions: [], + }); + await expect(evaluate('hook-1', 'merge-gate', {}, ctx)).resolves.toBe(true); + expect(streams.get('hooks')).toEqual([{ hook: 'merge-gate', step: 'hook-1', plugin: null, verdict: 'noop' }]); + }); + + it('runs implementations in lock order and AND-composes, stopping at the first false', async () => { + const { journal, streams } = fakeJournal(); + const order: string[] = []; + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [ + extension('first', async () => { order.push('first'); return true; }), + extension('second', async () => { order.push('second'); return false; }), + extension('third', async () => { order.push('third'); return true; }), + ], + }); + await expect(evaluate('hook-1', 'merge-gate', { owner: 'o' }, ctx)).resolves.toBe(false); + expect(order).toEqual(['first', 'second']); + expect(streams.get('hooks')).toEqual([ + { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'pass' }, + { hook: 'merge-gate', step: 'hook-1', plugin: 'second', verdict: 'fail' }, + ]); + }); + + it('replays a recorded verdict and does not re-run the closure', async () => { + const { journal, streams } = fakeJournal(); + streams.set('hooks', [ + { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'fail', because: 'held', afterStep: 9 }, + ]); + let calls = 0; + let nextStep = 2; + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [extension('first', async () => { calls += 1; return true; })], + peekStep: () => nextStep, + restoreStep: (step) => { nextStep = step; }, + }); + await expect(evaluate('hook-1', 'merge-gate', {}, ctx)).resolves.toBe(false); + expect(calls).toBe(0); + expect(nextStep).toBe(9); + expect(streams.get('hooks')).toHaveLength(1); + }); + + it('does not persist cancellation as a failed plugin verdict', async () => { + const { journal, streams } = fakeJournal(); + const controller = new AbortController(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [extension('first', async () => await new Promise(() => {}))], + signal: controller.signal, + }); + const pending = evaluate('hook-1', 'merge-gate', {}, ctx); + controller.abort(new Error('run cancelled')); + await expect(pending).rejects.toThrow('run cancelled'); + expect(streams.get('hooks')).toBeUndefined(); + }); + + it('refuses a hook the base header does not declare', async () => { + const { journal } = fakeJournal(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], extensions: [], + }); + await expect(evaluate('hook-1', 'nope', {}, ctx)).rejects.toMatchObject({ + code: 'unsupported_verb', + message: expect.stringContaining('hook "nope" is not declared'), + }); + }); +}); diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index bee894829..acc8367da 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -159,9 +159,28 @@ describe('durable authored root', () => { flowPath: loaded.sourcePath, input: { topic: 'relay' }, surface, }); expect(metadata.sourceSha256).toMatch(/^[a-f0-9]{64}$/u); + expect(metadata.extensions).toEqual([]); expect(journal.peer.completions).toEqual([{ attempt: 1, reason: 'success' }]); }); + it('journals plugin digests from composed extensions', async () => { + const loaded = await fixture(); + const extension = { + name: 'babysitter', + digest: 'c'.repeat(64), + ref: `github:AgentWorkforce/flows@${'a'.repeat(40)}#examples/babysitter`, + }; + const journal = new RootJournal(); + await executeDurableAuthoredFlow( + { ...loaded, extensions: [extension as LoadedAuthoredFlow['extensions'][number]] }, + journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'with-plugin' }, + ); + const step = (journal.starts[0]!.spec as { steps: Array<{ instruction: string }> }).steps[0]!; + const metadata = JSON.parse(step.instruction) as AuthoredRootMetadata; + expect(metadata.extensions).toEqual([extension]); + }); + it('reconciles a lost start acknowledgement from the durable root result', async () => { const loaded = await fixture(); const journal = new RootJournal(); @@ -364,6 +383,7 @@ async function fixture( return { sourcePath, handle, getDefinition, surfaceAuthority: surface, graph: [{ path: sourcePath, handle, getDefinition, surfaceAuthority: surface, use: [] }], + extensions: [], }; } @@ -394,6 +414,7 @@ function spawnedEntry(loaded: LoadedAuthoredFlow): Record { sourceSha256: '76fd521c5bda4f37b3c69a6ae3c5a97f0a2ba53d3d8b09d9cc9709f809f5a5a2', surface, }], + extensions: [], inputPresent: false, }; return { diff --git a/packages/sdk/tests/bundle.test.ts b/packages/sdk/tests/bundle.test.ts index 1c517eea7..14e72b6db 100644 --- a/packages/sdk/tests/bundle.test.ts +++ b/packages/sdk/tests/bundle.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest'; +import { existsSync } from 'node:fs'; import { mkdtemp, readFile, readdir, writeFile, rm, mkdir, symlink, stat, chmod, rename } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; @@ -6,7 +7,10 @@ import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; import { canonicalize } from '../src/canonical.js'; import { sealBundle, sha256, verifyBundle } from '../src/bundle.js'; +import { verifyBundlePluginLock } from '../src/bundle-extensions.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; import { buildFlow } from '../src/cli/build.js'; +import { SHA_A, entriesFromDirectory, fakeGithub } from './fake-github.js'; const sdk = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const repo = resolve(sdk, '../..'); @@ -27,7 +31,8 @@ function invoke(args: string[], cwd = repo, env: NodeJS.ProcessEnv = {}) { async function seal(out: string, env: NodeJS.ProcessEnv = { FLOWS_BUILD_KEY: key }, warn = (_: string) => {}) { return sealBundle({ name: 'example', repo: out, out, env, warn, files: [ { path: 'spec.canonical.json', data: canonicalize({ name: 'example' }) }, - { path: 'preflight.json', data: '{}' }, { path: 'lockfile.json', data: '{}' }, + { path: 'preflight.json', data: '{}' }, + { path: 'lockfile.json', data: canonicalize({ version: 2, plugins: [] }) }, ] }); } afterEach(async () => { await Promise.all(temporary.splice(0).map(path => rm(path, { force: true, recursive: true }))); }); @@ -218,6 +223,54 @@ describe('immutable bundles', () => { expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); }); + it('writes lockfile.json v2 with no plugins for a project that declares none', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'hello.yaml'), await readFile(join(repo, 'testdata/hello-deterministic.flow.yaml'))); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + expect(JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8'))).toEqual({ plugins: [], version: 2 }); + await verifyBundlePluginLock(bundle); + }); + + it('never accepts plugin payloads under a legacy npm lock', async () => { + const bundle = await temp(); + await writeFile(join(bundle, 'lockfile.json'), JSON.stringify({ lockfileVersion: 3, packages: {} })); + await verifyBundlePluginLock(bundle); + await mkdir(join(bundle, 'plugins/example'), { recursive: true }); + await writeFile(join(bundle, 'plugins/example/entry.js'), 'export default true;\n'); + await expect(verifyBundlePluginLock(bundle)).rejects.toThrow('legacy locks cannot authenticate plugins/ payloads'); + }); + + it('seals materialized flow-extension files under plugins// and verifies them', async () => { + const cwd = await temp(); + const fixture = join(repo, 'testdata/plugins/extension-babysitter'); + const gh = fakeGithub({ + 'AgentWorkforce/flows': { + refs: { main: SHA_A }, + commits: { [SHA_A]: { entries: entriesFromDirectory(fixture, 'examples/babysitter') } }, + }, + }); + await writeFile(join(cwd, 'flows.json'), JSON.stringify({})); + const io = { stdout: () => {}, stderr: () => {} }; + expect(await addExtensionPlugin(`github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`, io, { + cwd, fetch: gh.fetch, now: () => new Date('2026-09-20T12:00:00Z'), versions: { sdk: '2.0.22', surface: '2.0.22' }, + })).toBe(0); + await writeFile(join(cwd, 'hello.yaml'), await readFile(join(repo, 'testdata/hello-deterministic.flow.yaml'))); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + const lock = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); + expect(lock.version).toBe(2); + expect(lock.plugins).toHaveLength(1); + expect(lock.plugins[0]).toMatchObject({ name: 'babysitter', kind: 'flow-extension', order: 1 }); + expect(existsSync(join(bundle, 'plugins/babysitter/flows-plugin.json'))).toBe(true); + expect(existsSync(join(bundle, 'plugins/babysitter/babysitter.flow.ts'))).toBe(true); + expect(existsSync(join(bundle, 'plugins/babysitter/manifest.json'))).toBe(true); + expect(sha256(await readFile(join(bundle, 'plugins/babysitter/manifest.json')))).toBe(lock.plugins[0].digest); + expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); + await verifyBundlePluginLock(bundle); + const tampered = join(bundle, 'plugins/babysitter/babysitter.flow.ts'); + await writeFile(tampered, `${await readFile(tampered, 'utf8')}\n// tampered\n`); + await expect(verifyBundle(bundle)).rejects.toThrow('babysitter.flow.ts'); + }); + it('refuses build-provable CLI resolution errors without environment probes', async () => { const cwd = await temp(); await writeFile(join(cwd, 'invalid.yaml'), JSON.stringify({ version: '0.1.0', name: 'invalid', steps: [ diff --git a/packages/sdk/tests/catalog-plugins.test.ts b/packages/sdk/tests/catalog-plugins.test.ts new file mode 100644 index 000000000..6ee33050a --- /dev/null +++ b/packages/sdk/tests/catalog-plugins.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const SHA = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const TIERS = new Set(['first-party', 'verified', 'community']); + +describe('catalog/plugins.json', () => { + const catalog = JSON.parse(readFileSync(resolve('../../catalog/plugins.json'), 'utf8')) as { + version: unknown; + plugins: Array>; + }; + + it('is version 1 with unique kebab-case plugin names', () => { + expect(catalog.version).toBe(1); + expect(Array.isArray(catalog.plugins)).toBe(true); + expect(catalog.plugins.length).toBeGreaterThan(0); + const names = catalog.plugins.map(p => p.name); + expect(names.every(n => typeof n === 'string' && NAME.test(n))).toBe(true); + expect(new Set(names).size).toBe(names.length); + }); + + it('records a fail-closed babysitter entry with a pinned sha and digest', () => { + const babysitter = catalog.plugins.find(p => p.name === 'babysitter'); + expect(babysitter).toMatchObject({ + source: { owner: 'AgentWorkforce', repo: 'flows', path: 'examples/babysitter' }, + tier: 'community', + base: ['software-factory'], + }); + expect(babysitter!.ref).toMatch(SHA); + expect(babysitter!.digest).toMatch(HEX64); + expect(String(babysitter!.description)).toContain('plugin_event_unroutable'); + expect(TIERS.has(String(babysitter!.tier))).toBe(true); + }); +}); diff --git a/packages/sdk/tests/cloud-deploy.test.ts b/packages/sdk/tests/cloud-deploy.test.ts index 696612f83..b9e8584af 100644 --- a/packages/sdk/tests/cloud-deploy.test.ts +++ b/packages/sdk/tests/cloud-deploy.test.ts @@ -118,6 +118,7 @@ describe('deployToCloud', () => { }); expect(body.handoffId).toMatch(/^flows-cli-[a-f0-9]{16}$/u); expect(body.source).toContain("flow<{ issue: { title: string }; approver: string }>('issue-triage'"); + expect(body.extensions).toBeUndefined(); expect(deployment).toMatchObject({ agentId: 'agent-1', status: 'listening', name: 'issue-triage', connected: [] }); expect(deployment.requirements.integrations.map(i => `${i.provider} (${i.detail})`)).toEqual(['github (--on github)', 'slack (--on slack)']); expect(deployment.sourceSha256).toMatch(/^[a-f0-9]{64}$/u); diff --git a/packages/sdk/tests/fake-github.ts b/packages/sdk/tests/fake-github.ts new file mode 100644 index 000000000..c7df1e70f --- /dev/null +++ b/packages/sdk/tests/fake-github.ts @@ -0,0 +1,70 @@ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import type { FetchLike } from '../src/plugin-github.js'; + +/** + * An in-memory stand-in for the three public GitHub reads the SDK performs: + * commit resolution, recursive tree listing, and raw blob download. Tests + * shape repositories directly (symlinks, submodules, oversize files, byte + * drift, truncated trees) so every refusal is exercised offline. + */ +export interface FakeEntry { path: string; data?: Buffer; mode?: string; type?: string; size?: number } +export interface FakeCommit { entries: FakeEntry[]; truncated?: boolean } +export interface FakeRepo { refs: Record; commits: Record } +export interface FakeGithub { fetch: FetchLike; calls: string[]; repos: Record } + +export const SHA_A = 'a'.repeat(40); +export const SHA_B = 'b'.repeat(40); + +/** Every file under `directory`, mounted at `mountPath/` inside the fake repository. */ +export function entriesFromDirectory(directory: string, mountPath: string): FakeEntry[] { + const entries: FakeEntry[] = []; + const walk = (dir: string): void => { + for (const name of readdirSync(dir).sort()) { + const full = join(dir, name); + if (statSync(full).isDirectory()) { walk(full); continue; } + const path = `${mountPath === '' ? '' : `${mountPath}/`}${relative(directory, full).split('\\').join('/')}`; + entries.push({ path, data: readFileSync(full) }); + } + }; + walk(directory); + return entries; +} + +export function fakeGithub(repos: Record): FakeGithub { + const calls: string[] = []; + const fetch: FetchLike = async (url) => { + calls.push(url); + const u = new URL(url); + const respond = (status: number, body: string | Uint8Array | null = null, type = 'text/plain'): Response => new Response(body, { status, headers: { 'content-type': type } }); + if (u.host === 'api.github.com') { + let m = /^\/repos\/([^/]+)\/([^/]+)\/commits\/(.+)$/.exec(u.pathname); + if (m) { + const repo = repos[`${m[1]}/${m[2]}`]; + const ref = decodeURIComponent(m[3]!); + const sha = repo?.refs[ref] ?? (repo?.commits[ref] ? ref : undefined); + return sha === undefined ? respond(404) : respond(200, sha); + } + m = /^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([0-9a-f]{40})$/.exec(u.pathname); + if (m) { + const commit = repos[`${m[1]}/${m[2]}`]?.commits[m[3]!]; + if (!commit) return respond(404); + const dirs = new Set(); + for (const e of commit.entries) { const parts = e.path.split('/'); for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join('/')); } + const tree = [ + ...[...dirs].map(path => ({ path, mode: '040000', type: 'tree', sha: SHA_B })), + ...commit.entries.map(e => ({ path: e.path, mode: e.mode ?? '100644', type: e.type ?? 'blob', sha: SHA_B, size: e.size ?? e.data?.length ?? 0 })), + ]; + return respond(200, JSON.stringify({ sha: m[3], tree, truncated: commit.truncated ?? false }), 'application/json'); + } + return respond(404); + } + if (u.host === 'raw.githubusercontent.com') { + const m = /^\/([^/]+)\/([^/]+)\/([0-9a-f]{40})\/(.+)$/.exec(u.pathname); + const entry = m && repos[`${m[1]}/${m[2]}`]?.commits[m[3]!]?.entries.find(e => e.path === decodeURIComponent(m[4]!)); + return entry?.data === undefined ? respond(404) : respond(200, new Uint8Array(entry.data), 'application/octet-stream'); + } + return respond(404); + }; + return { fetch, calls, repos }; +} diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts new file mode 100644 index 000000000..922d886d8 --- /dev/null +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -0,0 +1,303 @@ +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { deployToCloud, parseTriggerSource } from '../src/cloud-deploy.js'; +import { collectExtensionSubmissions } from '../src/flow-extension-submit.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; +import { checkAuthoredTriggers } from '../src/cli/check-triggers.js'; +import { runCli } from '../src/cli.js'; +import { readPluginLock } from '../src/plugin-lock.js'; +import { loadPlugins } from '../src/plugin-loader.js'; +import { pluginStoreDirectory } from '../src/plugin-store.js'; +import { preflightProviderTriggers } from '../src/provider-trigger-contract.js'; +import { SHA_A, SHA_B, entriesFromDirectory, fakeGithub, type FakeEntry } from './fake-github.js'; + +const fixtureRoot = resolve('../../testdata/plugins'); +const babysitter = entriesFromDirectory(join(fixtureRoot, 'extension-babysitter'), 'examples/babysitter'); +const manifestJson = JSON.parse(readFileSync(join(fixtureRoot, 'extension-babysitter/flows-plugin.json'), 'utf8')); +const REF = `github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`; +const versions = { sdk: '2.0.22', surface: '2.0.22' }; +const now = () => new Date('2026-09-20T12:00:00Z'); +const dirs: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); +}); + +const BASE = ` + import { flow, github } from '@relayflows/surface'; + export default flow('software-factory', { budget: { dollars: 10, wallclock: '1h' } }, async f => { f.done('success'); }) + .on(github.issues({ action: 'opened' }), async f => { f.done('success'); }); +`; + +/** A project the way an operator has one: flows.json, the base flow, and a resolvable surface. */ +function project(base = BASE) { + const cwd = mkdtempSync(join(tmpdir(), 'flow-compose-')); dirs.push(cwd); + mkdirSync(join(cwd, 'node_modules/@relayflows'), { recursive: true }); + symlinkSync(resolve('node_modules/@relayflows/surface'), join(cwd, 'node_modules/@relayflows/surface')); + writeFileSync(join(cwd, 'package.json'), '{"type":"module"}'); + writeFileSync(join(cwd, 'flows.json'), JSON.stringify({ cli: 'claude', executors: ['github'] })); + writeFileSync(join(cwd, 'software-factory.flow.ts'), base); + const messages: string[] = []; + const io = { stdout: (s: string) => messages.push(s), stderr: (s: string) => messages.push(s) }; + return { cwd, io, messages, text: () => messages.join('\n'), flow: join(cwd, 'software-factory.flow.ts') }; +} +function repo(entries: FakeEntry[]) { + return fakeGithub({ 'AgentWorkforce/flows': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries } } } }).fetch; +} +function variant(patch: (m: Record) => unknown, entry?: string): FakeEntry[] { + return babysitter.map(e => { + if (e.path.endsWith('flows-plugin.json')) return { ...e, data: Buffer.from(JSON.stringify(patch(structuredClone(manifestJson)))) }; + if (entry !== undefined && e.path.endsWith('babysitter.flow.ts')) return { ...e, data: Buffer.from(entry) }; + return e; + }); +} +async function install(p: ReturnType, entries: FakeEntry[] = babysitter, ref = REF, fetch = repo(entries)) { + expect(await addExtensionPlugin(ref, p.io, { cwd: p.cwd, fetch, now, versions })).toBe(0); + p.messages.length = 0; +} +const subscriptions = (loaded: Awaited>) => + loaded.getDefinition(loaded.handle).handlers.map(h => { + const f = h.trigger.kind === 'webhook' ? h.trigger.filter as { type: string; payload?: { action?: string } } : undefined; + return f === undefined ? h.trigger.kind : `${f.type}${f.payload?.action === undefined ? '' : `.${f.payload.action}`}`; + }); + +describe('composing flow extensions onto a base flow', () => { + it('appends the Babysitter handler surface after the base, in lock order, without touching the base definition', async () => { + const p = project(); + await install(p); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + expect(loaded.extensions.map(e => ({ name: e.name, ref: e.ref, handlers: e.handlers.length }))).toEqual([{ name: 'babysitter', ref: REF, handlers: 8 }]); + expect(subscriptions(loaded)).toEqual([ + 'issues.opened', + 'pull_request.opened', 'pull_request.synchronize', 'pull_request.reopened', 'pull_request.closed', + 'pull_request_review.submitted', 'pull_request_review.dismissed', 'check_run.completed', 'issue_comment.created', + ]); + const composed = loaded.getDefinition(loaded.handle); + expect(Object.isFrozen(composed) && Object.isFrozen(composed.handlers)).toBe(true); + // The base's own definition, as its surface copy holds it, is unchanged. + expect(loaded.graph[0]!.getDefinition(loaded.handle).handlers).toHaveLength(1); + expect(composed.name).toBe('software-factory'); + expect(composed.header).toBe(loaded.graph[0]!.getDefinition(loaded.handle).header); + expect(loaded.graph.map(node => node.handle.name)).toEqual(['software-factory', 'babysitter']); + // Compare canonical paths: the loader realpaths the root (macOS tmpdir is a + // symlink, /var → /private/var), and the store path derives from that root. + expect(realpathSync(loaded.graph[1]!.path)).toBe(realpathSync(join(pluginStoreDirectory(p.cwd, 'babysitter', loaded.extensions[0]!.digest), 'babysitter.flow.ts'))); + // Every composed subscription is one the surface registry can lower. + expect(preflightProviderTriggers(composed.handlers.map(h => h.trigger))).toEqual([]); + // The extension's own handle is not the root: asking for its definition goes to the surface, not the composition. + expect(loaded.getDefinition(loaded.extensions[0]!.handle).handlers).toHaveLength(8); + // Its graph node resolves through the accessor its own entry import returned, not the root's. + const node = loaded.graph[1]!; + expect(node.getDefinition).toBe(loaded.extensions[0]!.getDefinition); + expect(node.getDefinition(node.handle).name).toBe('babysitter'); + expect(node.getDefinition(node.handle).handlers).toHaveLength(8); + }); + it('loads the root alone with extensions: none, and helper loading ignores extension entries', async () => { + const p = project(); + await install(p); + const alone = await loadAuthoredFlow(p.flow, { extensions: 'none' }); + expect(alone.extensions).toEqual([]); + expect(subscriptions(alone)).toEqual(['issues.opened']); + expect(await loadPlugins(p.cwd)).toEqual([]); + }); + it('composes two extensions in declaration order, and the order is the lockfile order', async () => { + const second = variant(m => ({ ...m, name: 'second', triggers: [{ provider: 'github', event: 'issues', actions: ['closed'] }] }), + "import { flow, github } from '@relayflows/surface';\nexport default flow('second', async f => { f.done('success'); }).on(github.issues({ action: 'closed' }), async f => { f.done('success'); });\n") + .map(e => ({ ...e, path: e.path.replace('examples/babysitter', 'examples/second') })); + const fetch = fakeGithub({ 'AgentWorkforce/flows': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries: babysitter }, [SHA_B]: { entries: second } } } }).fetch; + const SECOND = `github:AgentWorkforce/flows@${SHA_B}#examples/second`; + const forward = project(); + await install(forward, babysitter, REF, fetch); + await install(forward, second, SECOND, fetch); + expect(readPluginLock(forward.cwd).plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter'], [2, 'second']]); + const loadedForward = await loadAuthoredFlow(forward.flow, { versions }); + expect(loadedForward.extensions.map(e => e.name)).toEqual(['babysitter', 'second']); + expect(subscriptions(loadedForward).at(-1)).toBe('issues.closed'); + const reverse = project(); + await install(reverse, second, SECOND, fetch); + await install(reverse, babysitter, REF, fetch); + const loadedReverse = await loadAuthoredFlow(reverse.flow, { versions }); + expect(loadedReverse.extensions.map(e => e.name)).toEqual(['second', 'babysitter']); + expect(subscriptions(loadedReverse).slice(0, 2)).toEqual(['issues.opened', 'issues.closed']); + }); + it('flows check reports the composition and keeps the composed triggers deliverable', async () => { + const p = project(); + await install(p); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(true); + expect(report.extensions).toEqual([{ name: 'babysitter', version: '0.1.0', ref: REF, digest: expect.stringMatching(/^[0-9a-f]{64}$/), handlers: 8, hooks: [] }]); + expect(report.requirements?.integrations.map(i => i.provider)).toContain('github'); + expect(report.requirements?.harnessUses).toContainEqual({ harness: 'claude', detail: 'plugin "babysitter"' }); + expect(await runCli(['check', p.flow], p.io)).toBe(0); + expect(p.text()).toContain(`EXTENSION babysitter@0.1.0 ${REF} sha256:`); + expect(p.text()).toContain('8 handler(s) composed after the base flow'); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + const submissions = await collectExtensionSubmissions(loaded); + expect(submissions).toHaveLength(1); + expect(submissions[0]).toMatchObject({ name: 'babysitter', ref: REF }); + expect(submissions[0]!.files.some(f => f.path === 'babysitter.flow.ts' && f.encoding === 'utf8')).toBe(true); + expect(submissions[0]!.files.reduce((n, f) => n + f.bytes, 0)).toBeGreaterThan(0); + }); + it('flows check probes extension preflight before reporting the project healthy', async () => { + const p = project(); + await install(p, variant(m => ({ + ...m, + preflight: { credentials: ['FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'], servers: [] }, + }))); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(false); + expect(report.diagnostics[0]).toMatchObject({ + kind: 'plugin_credential_missing', + message: expect.stringContaining('FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'), + }); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + await expect(collectExtensionSubmissions(loaded)).rejects.toMatchObject({ + code: 'invalid_input', + message: expect.stringContaining('FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'), + }); + }); + it('uses extension permissions for hosted deploy preflight and the deploy body', async () => { + const p = project(); + await install(p, variant(m => ({ + ...m, + permissions: { ...(m.permissions as object), integrations: ['linear'], harnesses: ['codex'], mcp: ['filesystem'] }, + preflight: { credentials: [], servers: [] }, + }))); + const calls: Array<{ path: string; body: unknown }> = []; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const path = new URL(String(input)).pathname; + calls.push({ path, body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined }); + if (path === '/api/v1/auth/whoami') { + return new Response(JSON.stringify({ currentWorkspace: { id: 'ws-1' } }), { status: 200 }); + } + if (path.endsWith('/integrations/github/status')) return new Response('{"ready":true}', { status: 200 }); + if (path.endsWith('/integrations/linear/status')) return new Response('{"ready":false}', { status: 200 }); + if (path === '/api/v1/flows/deploy') { + return new Response(JSON.stringify({ agentId: 'agent-1', status: 'draft' }), { status: 201 }); + } + return new Response('{}', { status: 404 }); + }); + vi.stubEnv('FLOWS_CLOUD_URL', 'https://cloud-contract.example'); + vi.stubEnv('FLOWS_CLOUD_TOKEN', 'test-token'); + const input = { + path: p.flow, repository: { owner: 'AgentWorkforce', name: 'flows' }, + sources: [parseTriggerSource('github')], approver: 'reviewer', + }; + await expect(deployToCloud(input)).rejects.toMatchObject({ + code: 'integration_not_connected', message: expect.stringContaining('plugin "babysitter"'), + }); + expect(calls.some(call => call.path === '/api/v1/flows/deploy')).toBe(false); + + const deployed = await deployToCloud({ ...input, draft: true }); + const body = calls.findLast(call => call.path === '/api/v1/flows/deploy')!.body; + expect(body).toMatchObject({ + requirements: { integrations: ['github', 'linear'], harnesses: ['codex'], mcp: ['filesystem'] }, + }); + expect(deployed.requirements).toMatchObject({ + integrations: [ + { provider: 'github', from: 'source' }, + { provider: 'linear', from: 'extension', detail: 'plugin "babysitter"' }, + ], + harnesses: ['codex'], mcp: ['filesystem'], + }); + }); +}); + +describe('composition fails closed', () => { + it('refuses a tampered store before importing the entry', async () => { + const p = project(); + await install(p); + const store = pluginStoreDirectory(p.cwd, 'babysitter', readPluginLock(p.cwd).plugins[0]!.digest); + writeFileSync(join(store, 'babysitter.flow.ts'), "throw new Error('the entry must not be imported before the digest check');"); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_source_drift' }); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(false); + expect(report.diagnostics[0]).toMatchObject({ kind: 'plugin_source_drift' }); + }); + it('refuses when flows.json and the lockfile disagree', async () => { + const p = project(); + await install(p); + writeFileSync(join(p.cwd, 'flows.lock.json'), JSON.stringify({ version: 2, plugins: [] })); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_lock_invalid' }); + }); + it('refuses a runtime the extension declares itself incompatible with', async () => { + const p = project(); + await install(p); + await expect(loadAuthoredFlow(p.flow, { versions: { sdk: '2.0.22', surface: '3.0.0' } })).rejects.toMatchObject({ code: 'plugin_incompatible', message: expect.stringContaining('requires surface ^2.0.22') }); + }); + it.each([ + ['a base flow it does not extend', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'other-flow', version: '*' }] } }), 'plugin_incompatible', 'not "software-factory"'], + ['a base version range the unversioned base cannot satisfy', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^1.0.0' }] } }), 'plugin_incompatible', 'declares no version'], + ['a budget ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 11 } } }), 'plugin_incompatible', 'above the base flow'], + ['a wallclock ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { wallclock: '2h' } } }), 'plugin_incompatible', 'wallclock ceiling'], + ])('refuses %s', async (_, patch, code, message) => { + const p = project(); + await install(p, variant(patch)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code, message: expect.stringContaining(message) }); + }); + const HOOK_ENTRY = "import { flow, github } from '@relayflows/surface';\nexport const hooks = { 'merge-gate': async () => true };\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); });\n"; + it('refuses a hook export that the manifest does not declare', async () => { + const p = project(); + await install(p, variant(m => m, HOOK_ENTRY)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ + code: 'plugin_manifest_invalid', message: expect.stringContaining('does not match exported hooks'), + }); + }); + it('refuses a hook the base header does not declare', async () => { + const p = project(); + await install(p, variant(m => ({ ...m, extends: { handlers: true, hooks: ['merge-gate'] } }), HOOK_ENTRY)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ + code: 'plugin_incompatible', message: expect.stringContaining('hook merge-gate is not declared'), + }); + }); + it('composes a declared hook when the base header names it and reads header.version for compat', async () => { + const p = project(` + import { flow, github } from '@relayflows/surface'; + export default flow('software-factory', { version: '2.0.22', hooks: ['merge-gate'], budget: { dollars: 10, wallclock: '1h' } }, async f => { f.done('success'); }) + .on(github.issues({ action: 'opened' }), async f => { f.done('success'); }); + `); + await install(p, variant(m => ({ + ...m, + extends: { handlers: true, hooks: ['merge-gate'] }, + compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^2.0.0' }] }, + }), HOOK_ENTRY)); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + expect(Object.keys(loaded.extensions[0]!.hooks)).toEqual(['merge-gate']); + expect(loaded.getDefinition(loaded.handle).header).toMatchObject({ version: '2.0.22', hooks: ['merge-gate'] }); + }); + it.each([ + ['an entry subscribing beyond its manifest', undefined, + "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); }).on(github.pull_request('labeled'), async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'subscribes to github pull_request.labeled'], + ['an entry with a schedule handler', undefined, + "import { flow, schedule } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(schedule.every('1h'), async f => { f.done('success'); });\n", + 'plugin_unsupported', 'schedule trigger'], + ['an entry with a generic webhook handler', undefined, + "import { flow, webhook } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(webhook('deploys', { provider: 'aws' }), async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'not a provider subscription'], + ['an entry that declares no handlers although the manifest says it does', undefined, + "import { flow } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'declares no .on() handlers'], + ['an entry that uses the use: header', undefined, + "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', { use: ['./other.flow.ts'] }, async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); });\n", + 'plugin_unsupported', 'entry header use'], + ['an entry that does not default-export flow()', undefined, + "export default { name: 'forged' };\n", + 'plugin_manifest_invalid', 'did not load'], + ])('refuses %s', async (_, __, entry, code, message) => { + const p = project(); + await install(p, variant(m => m, entry)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code, message: expect.stringContaining(message) }); + }); + it('never composes an event the surface registry cannot lower, even if an entry asks for it', async () => { + // The manifest gate refuses ready_for_review at install; an entry alone cannot smuggle it past the manifest. + const p = project(); + const entries = variant(m => m, "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('ready_for_review'), async f => { f.done('success'); });\n"); + await install(p, entries); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_manifest_invalid', message: expect.stringContaining('pull_request.ready_for_review') }); + }); +}); diff --git a/packages/sdk/tests/flow-requirements.test.ts b/packages/sdk/tests/flow-requirements.test.ts index 184d38b70..a5ffd7fd3 100644 --- a/packages/sdk/tests/flow-requirements.test.ts +++ b/packages/sdk/tests/flow-requirements.test.ts @@ -6,7 +6,9 @@ import { getFlowDefinition } from '@relayflows/surface/runtime'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runCli } from '../src/cli.js'; import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; -import { describeFlowRequirements, flowRequirements, harnessFromCli } from '../src/flow-requirements.js'; +import { + describeFlowRequirements, flowRequirements, harnessFromCli, mergeFlowExtensionRequirements, +} from '../src/flow-requirements.js'; import { compileSpec } from '../src/compile.js'; import type { FlowSpec } from '../src/spec.js'; @@ -88,6 +90,24 @@ describe('flowRequirements on an authored definition', () => { .toEqual([{ provider: 'slack', from: 'tools', detail: 'tools.slack' }]); }); + it('merges manifest-only extension requirements in declaration order', () => { + const base = flowRequirements(getFlowDefinition(flow('base', { tools: { slack: true } }, async () => {}))); + expect(mergeFlowExtensionRequirements(base, [{ + name: 'babysitter', + permissions: { + integrations: ['slack', 'github'], harnesses: ['codex'], mcp: ['filesystem'], + }, + }])).toEqual({ + integrations: [ + { provider: 'slack', from: 'tools', detail: 'tools.slack' }, + { provider: 'github', from: 'extension', detail: 'plugin "babysitter"' }, + ], + harnesses: ['codex'], + harnessUses: [{ harness: 'codex', detail: 'plugin "babysitter"' }], + mcp: ['filesystem'], + }); + }); + it('reads a compiled spec through its steps and named agents', () => { const spec: FlowSpec = { version: '0.1.0', name: 'yaml', cli: 'gemini', agents: { drafter: { cli: 'codex', model: 'gpt-5' } }, steps: [ { id: 'a', type: 'deterministic', command: 'true' } as never, diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts new file mode 100644 index 000000000..47c16c2af --- /dev/null +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -0,0 +1,459 @@ +import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { addPlugin } from '../src/cli/add.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; +import { parsePluginArgs, runPluginCommand, verifyPlugins } from '../src/cli/plugin.js'; +import { runCli } from '../src/cli.js'; +import { validateFlowExtensionManifest } from '../src/flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha } from '../src/plugin-github.js'; +import { loadPlugins } from '../src/plugin-loader.js'; +import { parsePluginLock, readPluginLock, reconcileDeclaredExtensions } from '../src/plugin-lock.js'; +import { validatePluginManifest } from '../src/plugin-manifest.js'; +import { canonicalPluginRef, parseCanonicalPluginRef, parsePluginSource } from '../src/plugin-source.js'; +import { pluginStoreDirectory } from '../src/plugin-store.js'; +import { satisfiesRange } from '../src/semver-range.js'; +import { SHA_A, SHA_B, entriesFromDirectory, fakeGithub, type FakeEntry } from './fake-github.js'; + +const fixtureRoot = resolve('../../testdata/plugins'); +const babysitter = entriesFromDirectory(join(fixtureRoot, 'extension-babysitter'), 'examples/babysitter'); +const manifestJson = JSON.parse(readFileSync(join(fixtureRoot, 'extension-babysitter/flows-plugin.json'), 'utf8')); +const REF = `github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`; +const versions = { sdk: '2.0.22', surface: '2.0.22' }; +const now = () => new Date('2026-09-20T12:00:00Z'); +const dirs: string[] = []; +afterEach(() => { dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); vi.unstubAllEnvs(); }); + +function github(entries: FakeEntry[] = babysitter, extra: Partial<{ truncated: boolean }> = {}) { + return fakeGithub({ 'AgentWorkforce/flows': { refs: { 'feat/babysitter-v2': SHA_A, 'v0.1.0': SHA_A, main: SHA_B }, commits: { [SHA_A]: { entries, ...extra }, [SHA_B]: { entries: [] } } } }); +} +function project(config: unknown = {}) { + const cwd = mkdtempSync(join(tmpdir(), 'plugin-ext-')); dirs.push(cwd); + writeFileSync(join(cwd, 'flows.json'), JSON.stringify(config)); + const messages: string[] = []; + const io = { stdout: (s: string) => messages.push(s), stderr: (s: string) => messages.push(s) }; + return { cwd, io, messages, text: () => messages.join('\n') }; +} +function withManifest(patch: (m: Record) => unknown): FakeEntry[] { + return babysitter.map(e => e.path.endsWith('flows-plugin.json') ? { ...e, data: Buffer.from(JSON.stringify(patch(structuredClone(manifestJson)))) } : e); +} + +describe('plugin source references', () => { + it('parses the three accepted spellings to one shape', () => { + const expected = { host: 'github', owner: 'AgentWorkforce', repo: 'flows', ref: 'feat/babysitter-v2', path: 'examples/babysitter' }; + expect(parsePluginSource('github:AgentWorkforce/flows@feat/babysitter-v2#examples/babysitter')).toEqual(expected); + expect(parsePluginSource('https://github.com/AgentWorkforce/flows/tree/v0.1.0/examples/babysitter')).toEqual({ ...expected, ref: 'v0.1.0' }); + expect(parsePluginSource('https://github.com/AgentWorkforce/flows.git/blob/v0.1.0/examples/babysitter/')).toEqual({ ...expected, ref: 'v0.1.0' }); + expect(parsePluginSource('github:o/r@main')).toMatchObject({ path: '' }); + // The URL form cannot tell a slash in the ref from a path segment; that is + // what the github: form is for. Documented, not guessed. + expect(parsePluginSource('https://github.com/AgentWorkforce/flows/tree/feat/babysitter-v2/examples/babysitter')).toMatchObject({ ref: 'feat', path: 'babysitter-v2/examples/babysitter' }); + }); + it.each([ + 'github:o/r@main#../etc', 'github:o/r@main#a/../b', 'github:o/r@main#/abs', 'github:o/r@../x', 'github:o/r@main#a\\b', + 'https://github.com/o/r/tree/main/x?token=1', 'https://user:pw@github.com/o/r/tree/main/x', 'https://gitlab.com/o/r/tree/main/x', + 'github:o/r', 'github:o/r@main#examples/flows-plugin.json', 'github:-bad/r@main', 'github:o/r@main.lock', + 'https://github.com/o/r/tree/main/%E0%A4%A', + ])('refuses %s', input => { + expect(() => parsePluginSource(input)).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + }); + it('persists only the canonical sha form', () => { + expect(canonicalPluginRef({ host: 'github', owner: 'o', repo: 'r', ref: SHA_A, sha: SHA_A, path: '' })).toBe(`github:o/r@${SHA_A}`); + expect(parseCanonicalPluginRef(REF).sha).toBe(SHA_A); + expect(() => parseCanonicalPluginRef('github:o/r@main#x')).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + expect(() => parseCanonicalPluginRef(`https://github.com/o/r/tree/${SHA_A}/x`)).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + }); +}); + +describe('semver ranges', () => { + it.each([ + ['2.0.22', '^2.0.22', true], ['2.9.0', '^2.0.22', true], ['3.0.0', '^2.0.22', false], ['2.0.21', '^2.0.22', false], + ['2.0.30', '~2.0.22', true], ['2.1.0', '~2.0.22', false], ['5.0.0', '>=2.0.0', true], ['2.5.0', '>=2.0.0 <2.5.0', false], + ['2.0.22', '2.0.22', true], ['2.0.23', '2.0.22', false], ['0.0.9', '*', true], ['0.1.5', '^0.1.0', true], ['0.2.0', '^0.1.0', false], + ['2.0.22', 'latest', false], ['x', '*', false], + ['1.0.0-alpha.10', '>=1.0.0-alpha.2', true], ['1.0.0-alpha.2', '>=1.0.0-alpha.10', false], + ])('%s satisfies %s → %s', (version, range, ok) => { expect(satisfiesRange(version, range)).toBe(ok); }); +}); + +describe('flows add ', () => { + it('resolves a branch to its commit, materializes, and records flows.json plus the lockfile', async () => { + const gh = github(); const p = project({ cli: 'claude' }); + expect(await addPlugin('github:AgentWorkforce/flows@feat/babysitter-v2#examples/babysitter', p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + const config = JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')); + expect(config).toEqual({ cli: 'claude', plugins: [REF] }); + const lock = readPluginLock(p.cwd); + expect(lock.plugins).toHaveLength(1); + const [entry] = lock.plugins; + expect(entry).toMatchObject({ name: 'babysitter', kind: 'flow-extension', version: '0.1.0', order: 1, resolvedAt: '2026-09-20T12:00:00.000Z', source: { host: 'github', owner: 'AgentWorkforce', repo: 'flows', sha: SHA_A, path: 'examples/babysitter' } }); + expect(entry!.digest).toMatch(/^[0-9a-f]{64}$/); + const store = pluginStoreDirectory(p.cwd, 'babysitter', entry!.digest); + expect(existsSync(join(store, 'flows-plugin.json'))).toBe(true); + expect(existsSync(join(store, 'babysitter.flow.ts'))).toBe(true); + expect(existsSync(join(store, 'manifest.json'))).toBe(true); + expect(p.text()).toContain(`Added babysitter@0.1.0 (flow-extension) from ${REF}`); + expect(p.text()).toContain('events: github pull_request[opened,synchronize,reopened,closed]; github pull_request_review[submitted,dismissed]; github check_run[completed]; github issue_comment[created]'); + expect(p.text()).toContain('writes (declared, unenforced): github:pull_request:comment'); + expect(p.text()).toContain('recorded in flows.json and flows.lock.json'); + expect(gh.calls.some(url => url.includes('/commits/feat%2Fbabysitter-v2'))).toBe(true); + // A tag naming the same commit is a no-op re-add: no duplicate declaration, same lock entry. + expect(await addPlugin('https://github.com/AgentWorkforce/flows/tree/v0.1.0/examples/babysitter', p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + expect(readPluginLock(p.cwd)).toEqual(lock); + // And a sha input is accepted as-is. + expect(await addPlugin(REF, p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + }); + it('keeps the digest stable across identical installs and distinct across content changes', async () => { + const gh = github(); const a = project(); const b = project(); + await addExtensionPlugin(REF, a.io, { cwd: a.cwd, fetch: gh.fetch, now, versions }); + await addExtensionPlugin(REF, b.io, { cwd: b.cwd, fetch: gh.fetch, now, versions }); + expect(readPluginLock(a.cwd)).toEqual(readPluginLock(b.cwd)); + const changed = github(withManifest(m => ({ ...m, description: 'changed' }))); + const c = project(); + await addExtensionPlugin(REF, c.io, { cwd: c.cwd, fetch: changed.fetch, now, versions }); + expect(readPluginLock(c.cwd).plugins[0]!.digest).not.toBe(readPluginLock(a.cwd).plugins[0]!.digest); + }); + it('recovers both crash points in the flows.json and lock transaction', async () => { + const gh = github(); + for (const lockAlreadyRenamed of [false, true]) { + const p = project({ cli: 'claude' }); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + const nextConfig = JSON.stringify({ cli: 'claude', plugins: [] }); + const nextLock = JSON.stringify({ version: 2, plugins: [] }); + writeFileSync(join(p.cwd, 'flows.json.tmp'), nextConfig); + writeFileSync(join(p.cwd, lockAlreadyRenamed ? 'flows.lock.json' : 'flows.lock.json.tmp'), nextLock); + + expect(reconcileDeclaredExtensions(p.cwd)).toEqual([]); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8'))).toEqual({ cli: 'claude', plugins: [] }); + expect(readPluginLock(p.cwd)).toEqual({ version: 2, plugins: [] }); + expect(existsSync(join(p.cwd, 'flows.json.tmp'))).toBe(false); + expect(existsSync(join(p.cwd, 'flows.lock.json.tmp'))).toBe(false); + } + + const aborted = project(); + writeFileSync(join(aborted.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + expect(readPluginLock(aborted.cwd)).toEqual({ version: 2, plugins: [] }); + expect(existsSync(join(aborted.cwd, 'flows.lock.json.tmp'))).toBe(false); + + const truncated = project({ cli: 'claude' }); + writeFileSync(join(truncated.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + writeFileSync(join(truncated.cwd, 'flows.json.tmp'), '{"plugins":'); + expect(readPluginLock(truncated.cwd)).toEqual({ version: 2, plugins: [] }); + expect(JSON.parse(readFileSync(join(truncated.cwd, 'flows.json'), 'utf8'))).toEqual({ cli: 'claude' }); + expect(existsSync(join(truncated.cwd, 'flows.json.tmp'))).toBe(false); + expect(existsSync(join(truncated.cwd, 'flows.lock.json.tmp'))).toBe(false); + }); + it.each([ + ['github:AgentWorkforce/flows@nope#examples/babysitter', 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@main#examples/babysitter', 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@feat/babysitter-v2#examples', 'plugin_manifest_missing'], + [`github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`, 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@feat/babysitter-v2#../x', 'plugin_source_invalid'], + ])('refuses %s with %s and writes nothing', async (input, code) => { + const gh = github(); const p = project(); + expect(await addPlugin(input, p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(2); + expect(p.text()).toContain(`REFUSED [${code}]`); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8'))).toEqual({}); + expect(existsSync(join(p.cwd, 'flows.lock.json'))).toBe(false); + expect(existsSync(join(p.cwd, '.flows'))).toBe(false); + }); + it('refuses a plugin whose compat excludes this runtime', async () => { + const gh = github(); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions: { sdk: '2.0.22', surface: '3.0.0' } })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_incompatible] babysitter requires surface ^2.0.22; this runtime has 3.0.0.'); + }); + it('refuses a manifest whose declared source is not where it was fetched from', async () => { + const gh = github(withManifest(m => ({ ...m, source: { host: 'github', owner: 'someone', repo: 'else', path: 'examples/babysitter' } }))); + const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_drift]'); + }); + it('refuses a second source under an already-installed name', async () => { + const gh = github(); const p = project(); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + gh.repos['other/fork'] = { refs: { main: SHA_A }, commits: { [SHA_A]: { entries: babysitter } } }; + expect(await addExtensionPlugin(`github:other/fork@main#examples/babysitter`, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(2); + expect(p.text()).toContain('already installed from AgentWorkforce/flows'); + }); +}); + +describe('bounded, verified fetches', () => { + const source = { host: 'github' as const, owner: 'AgentWorkforce', repo: 'flows', ref: SHA_A, sha: SHA_A, path: 'examples/babysitter' }; + const plus = (entry: FakeEntry) => [...babysitter, entry]; + it.each([ + ['a symlink', plus({ path: 'examples/babysitter/link', data: Buffer.from('x'), mode: '120000' }), 'plugin_path_invalid'], + ['a submodule', plus({ path: 'examples/babysitter/vendor', type: 'commit', mode: '160000' }), 'plugin_path_invalid'], + ['a traversal path', plus({ path: 'examples/babysitter/a/../b', data: Buffer.from('x') }), 'plugin_path_invalid'], + ['a backslash path', plus({ path: 'examples/babysitter/a\\b', data: Buffer.from('x') }), 'plugin_path_invalid'], + ['an oversize file', plus({ path: 'examples/babysitter/big.bin', data: Buffer.alloc(256_001) }), 'plugin_too_large'], + ['a byte-count mismatch', plus({ path: 'examples/babysitter/drift.txt', data: Buffer.from('abc'), size: 2 }), 'plugin_source_drift'], + ])('refuses %s', async (_, entries, code) => { + await expect(fetchGithubPlugin(source, github(entries).fetch)).rejects.toMatchObject({ code }); + }); + it('refuses a plugin that exceeds the total byte budget', async () => { + const entries = [...babysitter, ...Array.from({ length: 9 }, (_, i) => ({ path: `examples/babysitter/part-${i}.bin`, data: Buffer.alloc(250_000) }))]; + await expect(fetchGithubPlugin(source, github(entries).fetch)).rejects.toMatchObject({ code: 'plugin_too_large' }); + }); + it('refuses a truncated tree listing rather than installing a partial plugin', async () => { + await expect(fetchGithubPlugin(source, github(babysitter, { truncated: true }).fetch)).rejects.toMatchObject({ code: 'plugin_fetch_failed', message: expect.stringContaining('truncated') }); + }); + it('reports a private or missing repository as unresolved, never as a transport error', async () => { + await expect(resolveGithubSha({ ...source, ref: 'main', owner: 'private', repo: 'repo' }, github().fetch)).rejects.toMatchObject({ code: 'plugin_source_unresolved' }); + const failing = async () => { throw new Error('ECONNRESET'); }; + await expect(resolveGithubSha(source, failing)).rejects.toMatchObject({ code: 'plugin_fetch_failed' }); + }); +}); + +describe('schema-2 manifest validation', () => { + it('accepts the worked Babysitter manifest and freezes it', () => { + const m = validateFlowExtensionManifest(manifestJson); + expect(m).toMatchObject({ schema: 2, kind: 'flow-extension', name: 'babysitter', entry: 'babysitter.flow.ts', extends: { handlers: true, hooks: [] } }); + expect(m.triggers).toHaveLength(4); + expect(Object.isFrozen(m) && Object.isFrozen(m.permissions) && Object.isFrozen(m.triggers)).toBe(true); + }); + it.each([ + ['an event the surface registry cannot lower', (m: Record) => ({ ...m, triggers: [{ provider: 'github', event: 'pull_request', actions: ['ready_for_review'] }] }), 'plugin_event_unroutable'], + ['labeled/unlabeled, which the registry lacks', (m: Record) => ({ ...m, triggers: [{ provider: 'github', event: 'pull_request', actions: ['labeled', 'unlabeled'] }] }), 'plugin_event_unroutable'], + ['an unknown provider', (m: Record) => ({ ...m, triggers: [{ provider: 'nope', event: 'x', actions: [] }] }), 'plugin_event_unroutable'], + ['an unknown kind', (m: Record) => ({ ...m, kind: 'banana' }), 'plugin_kind_invalid'], + ['schema 1 with the extension kind', (m: Record) => ({ ...m, schema: 1 }), 'plugin_manifest_invalid'], + ['verbs on a flow extension', (m: Record) => ({ ...m, verbs: [{ namespace: 'x', method: 'y', lowersTo: 'effect', args: {} }] }), 'plugin_manifest_invalid'], + ['an unknown top-level field', (m: Record) => ({ ...m, extra: 1 }), 'plugin_manifest_invalid'], + ['a helper- name', (m: Record) => ({ ...m, name: 'helper-x' }), 'plugin_manifest_invalid'], + ['a non-semver version', (m: Record) => ({ ...m, version: 'v1' }), 'plugin_manifest_invalid'], + ['a malformed compat range', (m: Record) => ({ ...m, compat: { ...(m.compat as object), surface: 'latest' } }), 'plugin_manifest_invalid'], + ['a traversal entry', (m: Record) => ({ ...m, entry: '../x.flow.ts' }), 'plugin_manifest_invalid'], + ['neither handlers nor hooks', (m: Record) => ({ ...m, extends: { handlers: false, hooks: [] } }), 'plugin_manifest_invalid'], + ['an unknown harness', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), harnesses: ['cursor'] } }), 'plugin_manifest_invalid'], + ['a malformed write class', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), writes: ['github'] } }), 'plugin_manifest_invalid'], + ['a non-positive budget', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 0 } } }), 'plugin_manifest_invalid'], + ['a config that is not a JSON Schema', (m: Record) => ({ ...m, config: { type: 'not-a-type' } }), 'plugin_manifest_invalid'], + ['a missing preflight', (m: Record) => { const { preflight, ...rest } = m; void preflight; return rest; }, 'plugin_preflight_missing'], + ])('refuses %s', (_, patch, code) => { + expect(() => validateFlowExtensionManifest(patch(structuredClone(manifestJson)))).toThrow(expect.objectContaining({ code })); + }); + it('keeps the helper validator helper-only and the extension validator extension-only', () => { + expect(() => validatePluginManifest(manifestJson)).toThrow(expect.objectContaining({ code: 'plugin_kind_invalid' })); + const helper = JSON.parse(readFileSync(join(fixtureRoot, 'helper-datadog/flows-plugin.json'), 'utf8')); + expect(validatePluginManifest(helper).name).toBe('helper-datadog'); + expect(validatePluginManifest({ ...helper, kind: 'helper', schema: 1 }).name).toBe('helper-datadog'); + expect(() => validatePluginManifest({ ...helper, schema: 2 })).toThrow(expect.objectContaining({ code: 'plugin_manifest_invalid' })); + expect(() => validateFlowExtensionManifest(helper)).toThrow(expect.objectContaining({ code: 'plugin_kind_invalid' })); + }); +}); + +describe('flows plugin list / verify', () => { + async function installed() { + const gh = github(); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + p.messages.length = 0; + return { gh, p, digest: readPluginLock(p.cwd).plugins[0]!.digest }; + } + it('lists in composition order and verifies locally and remotely', async () => { + const { gh, p, digest } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'list', json: false }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toBe(`1. babysitter@0.1.0 ${REF} sha256:${digest}`); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'list', json: true }, p.io, { cwd: p.cwd })).toBe(0); + expect(JSON.parse(p.text()).plugins[0]).toMatchObject({ name: 'babysitter', ref: REF, digest }); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: true }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toContain('remote skipped'); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: true, offline: false }, p.io, { cwd: p.cwd, fetch: gh.fetch })).toBe(0); + expect(JSON.parse(p.text())).toEqual({ ok: true, plugins: [{ name: 'babysitter', ref: REF, digest, remote: 'verified' }] }); + }); + it('refuses when GitHub serves different bytes at the pinned commit', async () => { + const { p } = await installed(); + const drifted = github(withManifest(m => ({ ...m, description: 'rewritten history' }))); + await expect(verifyPlugins(p.cwd, { offline: false, fetch: drifted.fetch })).rejects.toMatchObject({ code: 'plugin_source_drift', message: expect.stringContaining('GitHub now serves digest') }); + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: false }, p.io, { cwd: p.cwd, fetch: drifted.fetch })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_drift]'); + }); + it.each([ + ['an edited file', (store: string) => writeFileSync(join(store, 'babysitter.flow.ts'), '// tampered')], + ['a deleted file', (store: string) => rmSync(join(store, 'babysitter.flow.ts'))], + ['an added file', (store: string) => writeFileSync(join(store, 'extra.ts'), '')], + ['a symlink in place of a file', (store: string) => { rmSync(join(store, 'README.md')); symlinkSync('/etc/hostname', join(store, 'README.md')); }], + ['an edited manifest', (store: string) => writeFileSync(join(store, 'manifest.json'), '[]')], + ])('refuses the local store after %s', async (_, tamper) => { + const { p, digest } = await installed(); + tamper(pluginStoreDirectory(p.cwd, 'babysitter', digest)); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_source_drift' }); + }); + it('refuses when flows.json and the lockfile disagree', async () => { + const { p } = await installed(); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ plugins: [] })); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_lock_invalid', message: expect.stringContaining('does not declare') }); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ plugins: [REF, `github:o/r@${SHA_B}`] })); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_lock_invalid', message: expect.stringContaining('no entry') }); + }); + it('refuses a malformed lockfile', () => { + expect(() => parsePluginLock({ version: 1, plugins: [] })).toThrow(expect.objectContaining({ code: 'plugin_lock_invalid' })); + const entry = { name: 'x', kind: 'flow-extension', version: '1.0.0', source: { host: 'github', owner: 'o', repo: 'r', sha: SHA_A, path: '' }, digest: 'f'.repeat(64), manifestSha256: 'f'.repeat(64), order: 2, resolvedAt: '2026-09-20T00:00:00Z' }; + expect(() => parsePluginLock({ version: 2, plugins: [entry] })).toThrow(expect.objectContaining({ code: 'plugin_lock_invalid', message: expect.stringContaining('plugins[0]') })); + expect(parsePluginLock({ version: 2, plugins: [{ ...entry, order: 1 }] }).plugins[0]!.order).toBe(1); + expect(() => parsePluginLock({ version: 2, plugins: [{ ...entry, order: 1 }, { ...entry, order: 2 }] })).toThrow(expect.objectContaining({ message: expect.stringContaining('twice') })); + }); +}); + +describe('flows plugin remove / update', () => { + async function installed(entries: FakeEntry[] = babysitter) { + const gh = github(entries); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + p.messages.length = 0; + return { gh, p, digest: readPluginLock(p.cwd).plugins[0]!.digest }; + } + it('drops the declaration, rebuilds lock order, and deletes the store directory', async () => { + const sitter = babysitter.map(e => e.path.endsWith('flows-plugin.json') + ? { ...e, path: e.path.replace('examples/babysitter', 'examples/sitter'), data: Buffer.from(JSON.stringify({ ...manifestJson, name: 'sitter' })) } + : { ...e, path: e.path.replace('examples/babysitter', 'examples/sitter') }); + const gh = fakeGithub({ + 'AgentWorkforce/flows': { + refs: { 'feat/babysitter-v2': SHA_A, main: SHA_A }, + commits: { [SHA_A]: { entries: [...babysitter, ...sitter] } }, + }, + }); + const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + const sitterRef = `github:AgentWorkforce/flows@${SHA_A}#examples/sitter`; + expect(await addExtensionPlugin(sitterRef, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + const before = readPluginLock(p.cwd); + expect(before.plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter'], [2, 'sitter']]); + const babysitterDir = pluginStoreDirectory(p.cwd, 'babysitter', before.plugins[0]!.digest); + const sitterDir = pluginStoreDirectory(p.cwd, 'sitter', before.plugins[1]!.digest); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'remove', json: false, name: 'babysitter' }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toContain(`Removed babysitter@0.1.0 ${REF}`); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([sitterRef]); + const after = readPluginLock(p.cwd); + expect(after.plugins.map(e => [e.order, e.name])).toEqual([[1, 'sitter']]); + expect(existsSync(babysitterDir)).toBe(false); + expect(existsSync(sitterDir)).toBe(true); + }); + it('refuses to remove a name that is not installed', async () => { + const { p } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'remove', json: false, name: 'nope' }, p.io, { cwd: p.cwd })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_manifest_invalid] No flow-extension plugin named nope.'); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + }); + it('shows the permissions/events/budget diff and writes nothing without --yes', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ + ...m, + version: '0.2.0', + permissions: { ...(m.permissions as object), writes: ['github:pull_request:comment', 'github:issue:comment'], budget: { dollars: 12, wallclock: '1h' } }, + }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + updated.repos['AgentWorkforce/flows']!.refs.main = SHA_B; + const to = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: false, yes: false, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(2); + expect(p.text()).toContain(`Update babysitter ${REF} → ${to}`); + expect(p.text()).toContain('version: 0.1.0 → 0.2.0'); + expect(p.text()).toContain('+github:issue:comment'); + expect(p.text()).toContain('budget: $8 / 45m → $12 / 1h'); + expect(p.text()).toContain('REFUSED [plugin_manifest_invalid] Re-run with --yes to apply this update.'); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + expect(readPluginLock(p.cwd).plugins[0]!.digest).toBe(digest); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', digest))).toBe(true); + }); + it('applies --to with --yes, rewrites store/lock/flows.json, and drops the old store', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ ...m, version: '0.2.0', description: 'next' }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + updated.repos['AgentWorkforce/flows']!.refs.main = SHA_B; + const to = `github:AgentWorkforce/flows@main#examples/babysitter`; + const canonical = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: false, yes: true, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(0); + const lock = readPluginLock(p.cwd); + expect(lock.plugins).toHaveLength(1); + expect(lock.plugins[0]).toMatchObject({ name: 'babysitter', version: '0.2.0', order: 1, source: { sha: SHA_B } }); + expect(lock.plugins[0]!.digest).not.toBe(digest); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([canonical]); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', digest))).toBe(false); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', lock.plugins[0]!.digest))).toBe(true); + expect(p.text()).toContain(`Updated babysitter ${canonical}`); + }); + it('reports already-at when re-resolving the locked commit', async () => { + const { gh, p, digest } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: true, yes: false, name: undefined, to: undefined }, p.io, { + cwd: p.cwd, fetch: gh.fetch, now, versions, + })).toBe(0); + expect(JSON.parse(p.text())).toMatchObject({ ok: true, plugins: [{ name: 'babysitter', changed: false, digest }] }); + }); + it('emits the permissions diff in JSON without --yes and does not rewrite the lock', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ ...m, version: '0.2.0' }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + const to = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: true, yes: false, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(2); + expect(JSON.parse(p.messages.find(line => line.startsWith('{'))!)).toMatchObject({ ok: false, applied: false, code: 'plugin_manifest_invalid', plugins: [{ name: 'babysitter', changed: true }] }); + expect(readPluginLock(p.cwd).plugins[0]!.digest).toBe(digest); + }); + it('parses the new subcommands and refuses a malformed invocation', () => { + expect(parsePluginArgs(['remove', 'babysitter'])).toEqual({ command: 'plugin', sub: 'remove', json: false, name: 'babysitter' }); + expect(parsePluginArgs(['update', '--yes'])).toEqual({ command: 'plugin', sub: 'update', json: false, yes: true, name: undefined, to: undefined }); + expect(parsePluginArgs(['update', 'babysitter', '--to', 'github:o/r@main#x', '--yes', '--json'])) + .toEqual({ command: 'plugin', sub: 'update', json: true, yes: true, name: 'babysitter', to: 'github:o/r@main#x' }); + expect(parsePluginArgs(['remove'])).toBeUndefined(); + expect(parsePluginArgs(['update', '--to'])).toBeUndefined(); + expect(parsePluginArgs(['update', '--yes', '--yes'])).toBeUndefined(); + }); +}); + +describe('legacy helper plugins are untouched', () => { + it('recovers a pending extension transaction before adding a helper', async () => { + const p = project({ plugins: ['old-helper'] }); + writeFileSync(join(p.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + writeFileSync(join(p.cwd, 'flows.json.tmp'), JSON.stringify({ plugins: ['pending-helper'] })); + vi.stubEnv('DATADOG_API_KEY', 'test'); + const install = () => cpSync(join(fixtureRoot, 'helper-datadog'), join(p.cwd, 'node_modules/@flows/helper-datadog'), { recursive: true }); + expect(await addPlugin('helper-datadog', p.io, { cwd: p.cwd, install })).toBe(0); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins) + .toEqual(['pending-helper', '@flows/helper-datadog']); + }); + it('never contacts GitHub for a helper name and leaves the helper path to npm', async () => { + const gh = github(); const p = project(); + const install = vi.fn(() => { throw { stderr: 'offline' }; }); + expect(await addPlugin('helper-datadog', p.io, { cwd: p.cwd, install, extension: { fetch: gh.fetch } })).toBe(2); + expect(install).toHaveBeenCalledWith('@flows/helper-datadog', p.cwd); + expect(gh.calls).toEqual([]); + expect(p.text()).toContain('plugin_install_failed'); + }); + it('keeps the helper loader helper-only: extension entries are neither helpers nor unlisted packages', async () => { + const gh = github(); const p = project(); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + // Extensions are composed by the authored flow loader (flow-extension-compose.test.ts); here they are simply not helpers. + expect(await loadPlugins(p.cwd)).toEqual([]); + vi.stubEnv('DATADOG_API_KEY', 'test'); + cpSync(join(fixtureRoot, 'helper-datadog'), join(p.cwd, 'node_modules/@flows/helper-datadog'), { recursive: true }); + const config = JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ ...config, plugins: ['helper-datadog', ...config.plugins] })); + expect((await loadPlugins(p.cwd)).map(plugin => plugin.manifest.name)).toEqual(['helper-datadog']); + expect(readPluginLock(p.cwd).plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter']]); + }); + it('dispatches through the CLI: add refuses a bad reference offline, plugin list and verify run', async () => { + const p = project(); + expect(await runCli(['add', 'github:o/r@main#../x'], p.io)).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_invalid]'); + const previous = process.cwd(); + process.chdir(p.cwd); + try { + p.messages.length = 0; + expect(await runCli(['plugin', 'list'], p.io)).toBe(0); + expect(p.text()).toBe('No flow-extension plugins installed.'); + p.messages.length = 0; + expect(await runCli(['plugin', 'verify', '--offline', '--json'], p.io)).toBe(0); + expect(JSON.parse(p.text())).toEqual({ ok: true, plugins: [] }); + expect(await runCli(['plugin', 'nope'], p.io)).toBe(2); + } finally { process.chdir(previous); } + }); +}); diff --git a/packages/sdk/tests/preflight.test.ts b/packages/sdk/tests/preflight.test.ts index 6c5596f9c..a8fc0e68b 100644 --- a/packages/sdk/tests/preflight.test.ts +++ b/packages/sdk/tests/preflight.test.ts @@ -2,6 +2,8 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { addPlugin } from '../src/cli/add.js'; +import { runPluginCommand } from '../src/cli/plugin.js'; +import { SHA_A, fakeGithub, type FakeEntry } from './fake-github.js'; import type { PreflightFailureKind } from '../src/failure-kinds.js'; import { preflightHelpers } from '../src/preflight.js'; import { describe, expect, it } from 'vitest'; @@ -623,6 +625,49 @@ describe('preflight: CLI resolution and refusal predicates', () => { } } finally { rmSync(root, { recursive: true, force: true }); } } + // Flow-extension refusals (schema 2) exercise `flows add ` and + // `flows plugin verify` against an offline fake GitHub — the same public + // boundary a user hits, never a synthesized diagnostic. + { + const manifest = { + schema: 2, kind: 'flow-extension', name: 'ext', version: '0.1.0', entry: 'ext.flow.ts', + compat: { surface: '*', sdk: '*', base: [{ name: 'base', version: '*' }] }, + extends: { handlers: true, hooks: [] }, triggers: [], + permissions: { integrations: [], harnesses: [], mcp: [], writes: [] }, + preflight: { credentials: [], servers: [] }, + }; + const files = (m: unknown): FakeEntry[] => [ + { path: 'ext/flows-plugin.json', data: Buffer.from(JSON.stringify(m)) }, + { path: 'ext/ext.flow.ts', data: Buffer.from('export default 1;') }, + ]; + const repo = (entries: FakeEntry[]) => fakeGithub({ 'o/r': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries } } } }).fetch; + const extensionCases: { ref: string; fetch: import('../src/plugin-github.js').FetchLike }[] = [ + { ref: 'github:o/r@main#../x', fetch: repo(files(manifest)) }, + { ref: 'github:o/r@nope#ext', fetch: repo(files(manifest)) }, + { ref: 'github:o/r@main#ext', fetch: async () => { throw new Error('offline'); } }, + { ref: 'github:o/r@main#ext', fetch: repo([...files(manifest), { path: 'ext/link', data: Buffer.from('x'), mode: '120000' }]) }, + { ref: 'github:o/r@main#ext', fetch: repo([...files(manifest), { path: 'ext/big', data: Buffer.alloc(256_001) }]) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, kind: 'banana' })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, triggers: [{ provider: 'github', event: 'pull_request', actions: ['ready_for_review'] }] })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, compat: { ...manifest.compat, surface: '^1.0.0' } })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, source: { host: 'github', owner: 'someone', repo: 'else', path: 'ext' } })) }, + ]; + for (const { ref, fetch } of extensionCases) { + const root = mkdtempSync(join(tmpdir(), 'plugin-extension-taxonomy-')); + try { + writeFileSync(join(root, 'flows.json'), '{}'); + const io = { stdout() {}, stderr(line: string) { refusalKinds.push(line.match(/\[([^\]]+)\]/)![1] as PreflightFailureKind); } }; + expect(await addPlugin(ref, io, { cwd: root, extension: { fetch, versions: { sdk: '2.0.22', surface: '2.0.22' } } })).toBe(2); + } finally { rmSync(root, { recursive: true, force: true }); } + } + // `plugin_lock_invalid`: a declaration with no lockfile entry behind it. + const root = mkdtempSync(join(tmpdir(), 'plugin-lock-taxonomy-')); + try { + writeFileSync(join(root, 'flows.json'), JSON.stringify({ plugins: [`github:o/r@${SHA_A}#ext`] })); + const io = { stdout() {}, stderr(line: string) { refusalKinds.push(line.match(/\[([^\]]+)\]/)![1] as PreflightFailureKind); } }; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: true }, io, { cwd: root })).toBe(2); + } finally { rmSync(root, { recursive: true, force: true }); } + } // `plugin_unlisted` fires when a @flows/helper-* package is present in // node_modules but is missing from the declared plugins list — the loader // refuses to auto-load undeclared packages (plugin-loader.ts). Exercise it diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 3af09a9d3..ef8b02b6f 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -52,6 +52,7 @@ const RUN_ID = '01JABCDEFGHJKMNPQRSTVWXYZ0'; */ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: ParsedArgs['command'] }[] = [ { verb: 'add', argv: ['add', 'my-helper'], variant: 'add' }, + { verb: 'add', argv: ['add', 'github:AgentWorkforce/flows@main#examples/babysitter'], variant: 'add' }, { verb: 'answer', argv: ['answer', RUN_ID, 'human-1', 'yes'], variant: 'answer' }, { verb: 'answer', @@ -78,7 +79,7 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa verb: 'deploy', argv: ['deploy', 'review.flow.ts', '--repo', 'owner/name', '--on', 'github:label=review', '--approver', 'someone', '--name', 'review-listener', '--agents', 'claude,codex', '--draft', - '--no-connect', '--json'], + '--plugin', 'github:o/r@main#path', '--no-connect', '--json'], variant: 'cloud-deploy', }, { verb: 'deployments', argv: ['deployments', '--json'], variant: 'deployments' }, @@ -164,6 +165,18 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa '--poll-interval-ms', '1000', 'spec.json'], variant: 'tick', }, + { verb: 'plugin', argv: ['plugin', 'list'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'list', '--json'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'verify'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'verify', '--json', '--offline'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'remove', 'babysitter'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'remove', '--json', 'babysitter'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'update', '--yes'], variant: 'plugin' }, + { + verb: 'plugin', + argv: ['plugin', 'update', 'babysitter', '--to', 'github:o/r@main#path', '--yes', '--json'], + variant: 'plugin', + }, { verb: 'undeploy', argv: ['undeploy', 'dep_123'], variant: 'undeploy' }, { verb: 'undeploy', argv: ['undeploy', '--json', 'dep_123'], variant: 'undeploy' }, { verb: 'unschedule', argv: ['unschedule', 'sched_123'], variant: 'unschedule' }, diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index 38b48ce74..a9820ae76 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -72,6 +72,12 @@ export interface Ctx extends Helpers { */ human(question: string, options: { to: string }): Step; dispatch(flow: string, input: unknown): Promise; + /** + * Run every installed implementation of a named hook in lock order and + * AND-compose the booleans. With no implementations this is a journaled + * no-op that returns true. A name must appear in the flow header's `hooks`. + */ + hook(name: string, input: unknown): Step; done(reason: FlowCompletionReason): void; cloud: CloudHelper; memory: MemoryHelper; diff --git a/packages/surface/src/flow.ts b/packages/surface/src/flow.ts index fd06adeb5..bb0b84cd1 100644 --- a/packages/surface/src/flow.ts +++ b/packages/surface/src/flow.ts @@ -9,6 +9,10 @@ import { schedule } from "./schedule.js"; export interface FlowHeader { /** Relative paths to reusable authored flows composed by this body. */ use?: string[]; + /** Semver of this flow; plugins' `compat.base` ranges match against it. */ + version?: string; + /** Named hook points this body calls via `f.hook`; plugins may implement them. */ + hooks?: string[]; identity?: string; memory?: { script?: boolean; agent?: boolean }; budget?: string | { tokens?: number; dollars?: number; wallclock?: string }; @@ -20,6 +24,8 @@ export type FlowBody = (f: Ctx, input: Input) => Promise; export interface ReadonlyFlowHeader { readonly use?: readonly string[]; + readonly version?: string; + readonly hooks?: readonly string[]; readonly identity?: string; readonly memory?: Readonly<{ script?: boolean; agent?: boolean }>; readonly budget?: string | Readonly<{ tokens?: number; dollars?: number; wallclock?: string }>; @@ -163,15 +169,19 @@ function isStoredDefinition( && Object.isFrozen(value); } +const HEADER_FIELDS = [ + "use", + "version", + "hooks", + "identity", + "memory", + "budget", + "tools", + "workspace", +] as const; + function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { - const unknownFields = Object.keys(header).filter((field) => ![ - "use", - "identity", - "memory", - "budget", - "tools", - "workspace", - ].includes(field)); + const unknownFields = Object.keys(header).filter((field) => !(HEADER_FIELDS as readonly string[]).includes(field)); if (unknownFields.length > 0) { throw new TypeError(`flow header has unknown fields: ${unknownFields.join(", ")}`); } @@ -192,6 +202,8 @@ function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { }); return Object.freeze({ ...(header.use === undefined ? {} : { use: Object.freeze([...header.use]) }), + ...(header.version === undefined ? {} : { version: header.version }), + ...(header.hooks === undefined ? {} : { hooks: Object.freeze([...header.hooks]) }), ...(header.identity === undefined ? {} : { identity: header.identity }), ...(memory === undefined ? {} : { memory }), ...(header.budget === undefined ? {} : { budget: typeof header.budget === "string" ? header.budget : Object.freeze({ ...header.budget }) }), @@ -205,10 +217,14 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl assertHeaderObject(value, at); assertKnownKeys( value, - ["use", "identity", "memory", "budget", "tools", "workspace"], + HEADER_FIELDS, at, ); assertOptionalString(value, "identity", at); + assertOptionalString(value, "version", at); + if (value.version !== undefined && (value.version as string).trim().length === 0) { + throw new TypeError(`${at}.version: expected a nonempty version string`); + } if (value.budget !== undefined && typeof value.budget !== "string") { assertHeaderObject(value.budget, `${at}.budget`); assertKnownKeys(value.budget, ["tokens", "dollars", "wallclock"], `${at}.budget`); @@ -219,6 +235,18 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl } assertOptionalString(value, "workspace", at); assertOptionalStringArray(value, "use", at); + assertOptionalStringArray(value, "hooks", at); + if (value.hooks !== undefined) { + const hooks = value.hooks as string[]; + if (hooks.some((item) => item.trim().length === 0) || new Set(hooks).size !== hooks.length) { + throw new TypeError(`${at}.hooks: expected unique nonempty names`); + } + for (const hook of hooks) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(hook)) { + throw new TypeError(`${at}.hooks: expected kebab-case names`); + } + } + } if (value.use !== undefined) { for (const path of value.use as string[]) { if (!/^(?:\.\/|\.\.\/).+\.flow\.ts$/.test(path) || /[?#\\\\]/.test(path)) { diff --git a/packages/surface/tests/flow.test.ts b/packages/surface/tests/flow.test.ts index b919104f0..4147e17b1 100644 --- a/packages/surface/tests/flow.test.ts +++ b/packages/surface/tests/flow.test.ts @@ -62,6 +62,22 @@ describe("flow", () => { expect(Object.isFrozen(getFlowDefinition(definition).header.tools?.mcp)).toBe(true); }); + it("freezes version and hooks on the header", () => { + const hooks = ["pre-implement", "merge-gate"]; + const handle = flow("software-factory", { version: "2.0.22", hooks, budget: { dollars: 10 } }, async () => undefined); + hooks.push("later"); + expect(getFlowDefinition(handle).header).toMatchObject({ version: "2.0.22", hooks: ["pre-implement", "merge-gate"] }); + expect(Object.isFrozen(getFlowDefinition(handle).header.hooks)).toBe(true); + }); + + it.each([ + [{ hooks: ["MergeGate"] }, "header.hooks: expected kebab-case names"], + [{ hooks: ["merge-gate", "merge-gate"] }, "header.hooks: expected unique nonempty names"], + [{ version: "" }, "header.version: expected a nonempty version string"], + ])("refuses malformed version/hooks: %j", (header, message) => { + expect(() => flow("software-factory", header as FlowHeader, async () => undefined)).toThrow(message); + }); + it("validates raw header keys and nested values before cloning", () => { const invalidHeaders: { value: unknown; message: string }[] = [ { diff --git a/packages/ts-plugin/src/rules/header-keys.ts b/packages/ts-plugin/src/rules/header-keys.ts index 2b20eddbe..90af84766 100644 --- a/packages/ts-plugin/src/rules/header-keys.ts +++ b/packages/ts-plugin/src/rules/header-keys.ts @@ -3,7 +3,7 @@ import { DIAGNOSTICS, DIAGNOSTIC_SOURCE } from "../diagnostics"; // Mirrors assertFlowHeader in surface/src/flow.ts. SDK parity fixtures pin // these three closed lists without importing or executing author code in tsserver. -const HEADER_KEYS = ["identity", "memory", "budget", "tools", "workspace"]; +const HEADER_KEYS = ["identity", "memory", "budget", "tools", "workspace", "version", "hooks"]; const NESTED_KEYS: Record = { memory: ["script", "agent"], tools: ["relayfile", "mcp", "slack"], diff --git a/testdata/plugins/extension-babysitter/README.md b/testdata/plugins/extension-babysitter/README.md new file mode 100644 index 000000000..bccfa64d3 --- /dev/null +++ b/testdata/plugins/extension-babysitter/README.md @@ -0,0 +1,19 @@ +# extension-babysitter (offline fixture) + +The worked schema-2 `kind: "flow-extension"` manifest for Babysitter on +Software Garden, served to the SDK tests by a fake GitHub (see +`packages/sdk/tests/plugin-extension.test.ts` and +`tests/flow-extension-compose.test.ts`). It is a fixture, not an installable +example: the entry carries Babysitter's handler surface — one `.on()` per +declared subscription — over a body that only declines, so composition onto a +base flow can be proven without the real review body. `extends.hooks` is empty +because this fixture does not declare a hook; declared hooks are composed when +the base flow names them. The `merge-gate` hook from the design is not included +in this fixture. + +Babysitter's own subscription contract (branch `feat/babysitter-v2`) names +eleven GitHub subscriptions. Three of them — `pull_request.ready_for_review`, +`pull_request.labeled`, `pull_request.unlabeled` — are not in the surface +event registry (`providerEventTypes`), so a manifest declaring them is refused +with `plugin_event_unroutable`; this fixture lists only the eight the registry +can lower. Extending the registry is a separate change and is not claimed here. diff --git a/testdata/plugins/extension-babysitter/babysitter.flow.ts b/testdata/plugins/extension-babysitter/babysitter.flow.ts new file mode 100644 index 000000000..e27553f09 --- /dev/null +++ b/testdata/plugins/extension-babysitter/babysitter.flow.ts @@ -0,0 +1,17 @@ +// Offline fixture entry: the handler surface of Babysitter, one `.on()` per +// subscription the manifest declares, over a body that only declines. The real +// review body lives on the babysitter branch of AgentWorkforce/flows; this +// file exists so composition can be proven against the declared contract. +import { flow, github, type Ctx } from '@relayflows/surface'; + +async function babysit(f: Ctx): Promise { f.done('declined'); } + +export default flow('babysitter', { budget: { dollars: 8, wallclock: '45m' } }, babysit) + .on(github.pull_request('opened'), babysit) + .on(github.pull_request('synchronize'), babysit) + .on(github.pull_request('reopened'), babysit) + .on(github.pull_request('closed'), babysit) + .on(github.pull_request_review({ action: 'submitted' }), babysit) + .on(github.pull_request_review({ action: 'dismissed' }), babysit) + .on(github.check_run('completed'), babysit) + .on(github.issue_comment('created'), babysit); diff --git a/testdata/plugins/extension-babysitter/flows-plugin.json b/testdata/plugins/extension-babysitter/flows-plugin.json new file mode 100644 index 000000000..5f7afe7eb --- /dev/null +++ b/testdata/plugins/extension-babysitter/flows-plugin.json @@ -0,0 +1,125 @@ +{ + "schema": 2, + "kind": "flow-extension", + "name": "babysitter", + "version": "0.1.0", + "description": "Live-state PR babysitter: parallel review lenses, deterministic reconciliation, exact-head merge gate. Worked schema-2 example; the real flow lives on the babysitter branch of AgentWorkforce/flows.", + "compat": { + "surface": "^2.0.22", + "sdk": "^2.0.22", + "base": [ + { + "name": "software-factory", + "version": "*" + } + ] + }, + "entry": "babysitter.flow.ts", + "extends": { + "handlers": true, + "hooks": [] + }, + "triggers": [ + { + "provider": "github", + "event": "pull_request", + "actions": [ + "opened", + "synchronize", + "reopened", + "closed" + ] + }, + { + "provider": "github", + "event": "pull_request_review", + "actions": [ + "submitted", + "dismissed" + ] + }, + { + "provider": "github", + "event": "check_run", + "actions": [ + "completed" + ] + }, + { + "provider": "github", + "event": "issue_comment", + "actions": [ + "created" + ] + } + ], + "permissions": { + "integrations": [ + "github" + ], + "harnesses": [ + "claude" + ], + "mcp": [], + "writes": [ + "github:pull_request:comment" + ], + "budget": { + "dollars": 8, + "wallclock": "45m" + } + }, + "preflight": { + "credentials": [], + "servers": [ + "https://api.github.com" + ] + }, + "config": { + "type": "object", + "properties": { + "testCommand": { + "type": "string" + }, + "botLogin": { + "type": "string" + }, + "approvers": { + "type": "array", + "items": { + "type": "string" + } + }, + "organizations": { + "type": "array", + "items": { + "type": "string" + } + }, + "merge": { + "type": "boolean", + "default": false + }, + "skipLabels": { + "type": "array", + "items": { + "type": "string" + }, + "default": [ + "no-agent-relay-review" + ] + }, + "requiredChecks": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "testCommand", + "approvers" + ], + "additionalProperties": false + } +}