From 8075463edc3aea0ff3f8d870cf92b1cb234c8ec9 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 15:14:06 -0700 Subject: [PATCH 01/17] =?UTF-8?q?feat(sdk):=20schema-2=20flow-extension=20?= =?UTF-8?q?plugins=20from=20public=20GitHub=20=E2=80=94=20add,=20lock,=20v?= =?UTF-8?q?erify=20(P1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same flows-plugin.json now carries a second kind. `kind` absent stays the schema-1 helper plugin, byte-for-byte: npm @flows/helper-*, effect verbs, flows.json allowlisting, preflight. `"schema": 2, "kind": "flow-extension"` is a directory in a public GitHub repository whose entry default-exports flow() and declares handlers/hooks, triggers (validated against the surface event registry, refused with plugin_event_unroutable otherwise), permissions (declared-but-unenforced writes, budget ceiling), compat ranges, and the same mandatory preflight. `flows add github:/@#` resolves a branch, tag, or commit to a 40-hex sha through unauthenticated public reads, enumerates the tree at that commit (refusing symlinks, submodules, traversal, a truncated listing, files over 256 KB, plugins over 2 MB), downloads blobs pinned to the sha with byte-count checks, and computes the content digest with the same canonical payload manifest sealed bundles use (bundle.ts payloadManifest, now shared). Bytes land under .flows/plugins/@sha256:/; flows.json.plugins records only the canonical sha form; flows.lock.json v2 records name, version, source, digest, manifest hash, and declaration order. `flows plugin list` and `flows plugin verify [--offline]` read those back; any local or remote difference is plugin_source_drift, exit 2. Runtime composition is not in this slice: a project declaring a flow extension is refused at load time with plugin_unsupported, before any helper loads, so a base flow never silently runs without an extension it declared. Tests: offline fake GitHub covering branch/tag/sha resolution, idempotent re-add, digest stability, every refusal kind (also wired into the preflight exhaustiveness test), local-store tampering, lockfile/flows.json disagreement, legacy helper path untouched, CLI dispatch and verb-table drift guards. The worked Babysitter manifest is testdata/plugins/extension-babysitter (fixture, not an installable example); it lists the eight GitHub subscriptions the registry can lower and documents the three it cannot. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 19498b5b-4a5c-4096-a978-84d7082bd5a4 --- docs/SURFACE.md | 41 +++ packages/sdk/src/bundle.ts | 18 +- packages/sdk/src/cli-commands.ts | 17 +- packages/sdk/src/cli.ts | 7 + packages/sdk/src/cli/add-extension.ts | 112 +++++++ packages/sdk/src/cli/add.ts | 7 + packages/sdk/src/cli/plugin.ts | 86 +++++ packages/sdk/src/flow-extension-manifest.ts | 190 +++++++++++ packages/sdk/src/plugin-github.ts | 98 ++++++ packages/sdk/src/plugin-loader.ts | 6 + packages/sdk/src/plugin-lock.ts | 99 ++++++ packages/sdk/src/plugin-manifest.ts | 17 + packages/sdk/src/plugin-source.ts | 87 +++++ packages/sdk/src/plugin-store.ts | 90 ++++++ packages/sdk/src/semver-range.ts | 52 +++ packages/sdk/tests/fake-github.ts | 70 ++++ packages/sdk/tests/plugin-extension.test.ts | 299 ++++++++++++++++++ packages/sdk/tests/preflight.test.ts | 45 +++ packages/sdk/tests/relay-cli-surface.test.ts | 5 + .../plugins/extension-babysitter/README.md | 14 + .../extension-babysitter/babysitter.flow.ts | 5 + .../extension-babysitter/flows-plugin.json | 42 +++ 22 files changed, 1401 insertions(+), 6 deletions(-) create mode 100644 packages/sdk/src/cli/add-extension.ts create mode 100644 packages/sdk/src/cli/plugin.ts create mode 100644 packages/sdk/src/flow-extension-manifest.ts create mode 100644 packages/sdk/src/plugin-github.ts create mode 100644 packages/sdk/src/plugin-lock.ts create mode 100644 packages/sdk/src/plugin-source.ts create mode 100644 packages/sdk/src/plugin-store.ts create mode 100644 packages/sdk/src/semver-range.ts create mode 100644 packages/sdk/tests/fake-github.ts create mode 100644 packages/sdk/tests/plugin-extension.test.ts create mode 100644 testdata/plugins/extension-babysitter/README.md create mode 100644 testdata/plugins/extension-babysitter/babysitter.flow.ts create mode 100644 testdata/plugins/extension-babysitter/flows-plugin.json diff --git a/docs/SURFACE.md b/docs/SURFACE.md index b1c7f62ac..3c064cd59 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -562,6 +562,47 @@ plugin code bundling/pinning, declarative-flow plugin preflight, and restart recovery of an interrupted plugin effect. Plugin effects currently inherit the internal authored executor's child-run lifecycle, not a resumable authored root. +### Flow extensions: schema 2, `kind: "flow-extension"` + +The same `flows-plugin.json` file carries a second kind. A **helper** plugin +(`kind` absent) extends `Ctx` with verbs and installs from npm as above. A +**flow extension** (`"schema": 2, "kind": "flow-extension"`) is a directory in +a public GitHub repository whose `entry` default-exports `flow()` and declares +what it will contribute to a base flow — `extends.handlers` (its `.on()` +pairs), `extends.hooks` (named points the base calls), `triggers` (validated +against the surface event registry, refused with `plugin_event_unroutable` +otherwise), `permissions` (integrations, harnesses, mcp, declared-but-unenforced +`writes`, a budget ceiling), `compat` (semver ranges for surface and sdk, and +the base flows it extends), and the same mandatory `preflight`. Validation is +`packages/sdk/src/flow-extension-manifest.ts`; the worked Babysitter manifest is +`testdata/plugins/extension-babysitter/flows-plugin.json`. + +```text +flows add github:/@# # or https://github.com///tree// +flows plugin list [--json] +flows plugin verify [--json] [--offline] +``` + +`flows add` resolves the branch, tag, or commit to a 40-hex sha through +unauthenticated public GitHub reads (a private repository answers 404 and is +reported as `plugin_source_unresolved`), enumerates the tree at that commit — +refusing symlinks, submodules, traversal, a truncated listing, files over +256 KB, or plugins over 2 MB — downloads each blob pinned to the sha, checks +byte counts, and computes the content digest as the sha256 of the same +canonical `[{bytes,path,sha256}]` manifest a sealed bundle uses. The bytes are +materialized under `.flows/plugins/@sha256:/`; `flows.json.plugins` +gains the canonical `github:/@#` (a branch or tag is +never persisted); and `flows.lock.json` (version 2) records name, version, +source, digest, manifest hash, and the declaration order that will be the +composition order. `flows plugin verify` re-hashes the store against the lock +and, unless `--offline`, re-fetches the pinned commit; any difference is +`plugin_source_drift`, exit 2. + +Installing records a declaration; it does not enable execution. A project that +declares a flow extension is refused at run time with `plugin_unsupported` +until the handlers/hooks slice lands, so a base flow never silently runs +without the extension it was told it had. + ## 4. Build: the immutable bundle `flows build` seals a flow into a content-addressed, immutable bundle: canonical spec JSON, compiled TS with pinned deps, helper/plugin lockfile, assets, preflight declaration, identity signature — `flow@sha256:…`, pushed to a bucket/registry. `flows deploy` points a trigger at a digest; `flows run flow@sha256:…` executes from the bucket on any cell, no checkout. Preflight runs at build time for everything build-provable and again at deploy time for environment facts (credentials, workers, MCP servers). The working tree is for authoring; **production only ever runs digests.** diff --git a/packages/sdk/src/bundle.ts b/packages/sdk/src/bundle.ts index 470785f90..bbf986783 100644 --- a/packages/sdk/src/bundle.ts +++ b/packages/sdk/src/bundle.ts @@ -18,12 +18,24 @@ export function sha256(data: Uint8Array | string): string { return createHash('sha256').update(data).digest('hex'); } -function safePath(path: string): boolean { +/** A bundle-relative path: no empty, `.`, or `..` component, no backslash, NUL, or drive colon. */ +export function safePath(path: string): boolean { return path.length > 0 && !path.includes('\\') && !path.includes('\0') && path.split('/').every(part => part !== '' && part !== '.' && part !== '..') && !path.includes(':'); } +/** + * The canonical entry list whose sha256 is a payload's content digest. Shared + * by sealed flow bundles and materialized plugins so `@sha256:` means one + * thing everywhere: the digest of `[{bytes,path,sha256}]`, sorted by path. + */ +export function payloadManifest(files: readonly { path: string; data: Uint8Array }[]): string { + return canonicalize([...files] + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0) + .map(file => ({ path: file.path, sha256: sha256(file.data), bytes: file.data.length }))); +} + /** Manifest and identity are envelopes, excluded to avoid circular hashing. */ export async function sealBundle(options: BundleOptions): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(options.name)) { @@ -41,9 +53,7 @@ export async function sealBundle(options: BundleOptions): Promise { for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) { if (!paths.has(required)) throw new Error(`${required}: missing bundle file`); } - const manifest = canonicalize(files.map(file => ({ - path: file.path, sha256: sha256(file.data), bytes: file.data.length, - }))); + const manifest = payloadManifest(files); const digest = sha256(manifest); const out = resolve(options.out); const target = join(out, `${options.name}@sha256:${digest}`); diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index ed2ee8669..23d6237d2 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -104,8 +104,8 @@ const LOCAL_EXECUTION_OPTIONS = [ export const CLI_VERBS = [ { name: 'add', - description: 'Install a helper plugin into this project', - args: [{ name: 'helper', description: 'Helper name or @flows/', required: true }], + description: 'Install a helper plugin, or a flow-extension plugin from a public GitHub repository, into this project', + args: [{ name: 'plugin', description: 'Helper name, @flows/, github:/@#, or a github.com tree URL', required: true }], variants: ['add'], }, { @@ -199,6 +199,19 @@ export const CLI_VERBS = [ options: [DATA_DIR_OPTION], variants: ['observer'], }, + { + name: 'plugin', + description: 'Inspect the flow-extension plugins recorded in flows.lock.json', + subcommands: [ + { name: 'list', description: 'List installed flow extensions in composition order', options: [JSON_OPTION] }, + { + name: 'verify', + description: 'Re-hash .flows/plugins against the lockfile and, unless --offline, against the pinned commit on GitHub', + options: [JSON_OPTION, { flags: '--offline', description: 'Skip the GitHub re-fetch; check only the local store against the lockfile' }], + }, + ], + variants: ['plugin'], + }, { name: 'replay', description: 'Replay a finished run from its local journal', diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 75bfc1b1f..a11ea5dad 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node import { addPlugin } from './cli/add.js'; +import { parsePluginArgs, runPluginCommand, type PluginArgs } from './cli/plugin.js'; import { watchCheck } from './cli-watch.js'; import { checkHelperBody } from './cli/check-helper-body.js'; import { describeFlowRequirements } from './flow-requirements.js'; @@ -64,6 +65,7 @@ type CliExitCode = 0 | 1 | 2 | 3; */ export type ParsedArgs = | { command: 'add'; value: string } + | PluginArgs | ReplayArgs | StatusArgs | BuildArgs @@ -92,6 +94,9 @@ export type ParsedArgs = const USAGE = [ 'Usage:', 'flows add ', + 'flows add ', + 'flows plugin list [--json]', + 'flows plugin verify [--json] [--offline]', 'flows build [--out ] ', 'flows build --verify ', 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--no-connect] [--json]', @@ -192,6 +197,7 @@ export async function runCli( } if (parsed.command === 'add') return addPlugin(parsed.value, io); + if (parsed.command === 'plugin') return runPluginCommand(parsed, io); if (parsed.command === 'serve-webhook') { return withInterrupt(options.signal, (signal) => runServeWebhook(parsed, io, signal)); @@ -521,6 +527,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // parser, so the declared tree and the dispatched tree cannot drift apart. if (command === undefined || !CLI_VERB_NAMES.has(command)) return undefined; if (command === 'add') return args.length === 2 ? { command: 'add', value: args[1]! } : undefined; + if (command === 'plugin') return parsePluginArgs(args.slice(1)); if (command === 'replay') return parseReplayArgs(args.slice(1)); if (command === 'status') return parseStatusArgs(args.slice(1)); if (command === 'runs') return parseRunsArgs(args.slice(1)); diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts new file mode 100644 index 000000000..f21a542ce --- /dev/null +++ b/packages/sdk/src/cli/add-extension.ts @@ -0,0 +1,112 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { CliIo } from '../cli.js'; +import { sha256 } from '../bundle.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha, type FetchLike, type FetchedPlugin } from '../plugin-github.js'; +import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, writePluginLock } from '../plugin-lock.js'; +import { findPluginProject } from '../plugin-loader.js'; +import { PluginError } from '../plugin-manifest.js'; +import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; +import { materializePlugin } from '../plugin-store.js'; +import { satisfiesRange } from '../semver-range.js'; + +/** The versions a plugin's `compat` is checked against: this SDK and the surface it pins. */ +export function runtimeVersions(): { sdk: string; surface: string } { + const pkg = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')) as { version: string; dependencies: Record }; + return { sdk: pkg.version, surface: pkg.dependencies['@relayflows/surface']! }; +} + +export function assertCompatible(manifest: FlowExtensionManifest, versions: { sdk: string; surface: string }): void { + for (const [what, range, actual] of [['surface', manifest.compat.surface, versions.surface], ['sdk', manifest.compat.sdk, versions.sdk]] as const) { + if (!satisfiesRange(actual, range)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${what} ${range}; this runtime has ${actual}.`); + } +} + +/** Parse and validate the manifest inside a fetched plugin, checking that any self-declared source is the one it came from. */ +export function extensionManifestOf(plugin: FetchedPlugin): { manifest: FlowExtensionManifest; manifestSha256: string } { + const file = plugin.files.find(f => f.path === 'flows-plugin.json'); + if (file === undefined) throw new PluginError('plugin_manifest_missing', 'Plugin has no flows-plugin.json.'); + let input: unknown; + try { input = JSON.parse(file.data.toString('utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'flows-plugin.json is unreadable or invalid JSON.'); } + const manifest = validateFlowExtensionManifest(input); + const { source } = plugin; + if (manifest.source !== undefined && (manifest.source.owner !== source.owner || manifest.source.repo !== source.repo || manifest.source.path !== source.path + || (manifest.source.sha !== undefined && manifest.source.sha !== source.sha))) { + throw new PluginError('plugin_source_drift', `flows-plugin.json declares source ${manifest.source.owner}/${manifest.source.repo}#${manifest.source.path}, but it was fetched from ${source.owner}/${source.repo}#${source.path}.`); + } + if (!plugin.files.some(f => f.path === manifest.entry)) throw new PluginError('plugin_manifest_invalid', `entry ${manifest.entry} is not in the plugin.`); + return { manifest, manifestSha256: sha256(file.data) }; +} + +export function describeExtension(manifest: FlowExtensionManifest): string[] { + const p = manifest.permissions; + return [ + ` integrations: ${p.integrations.join(', ') || 'none'}; harnesses: ${p.harnesses.join(', ') || 'none'}; mcp: ${p.mcp.join(', ') || 'none'}`, + ` events: ${manifest.triggers.map(t => `${t.provider} ${t.event}[${t.actions.join(',')}]`).join('; ') || 'none'}`, + ` hooks: ${manifest.extends.hooks.join(', ') || 'none'}; handlers: ${manifest.extends.handlers ? 'yes' : 'no'}`, + ` writes (declared, unenforced): ${p.writes.join(', ') || 'none'}`, + ` budget: ${p.budget === undefined ? 'inherits base' : [p.budget.dollars === undefined ? '' : `$${p.budget.dollars}`, p.budget.wallclock ?? ''].filter(Boolean).join(' / ')}`, + ]; +} + +export interface AddExtensionOptions { + cwd?: string; + fetch?: FetchLike; + now?: () => Date; + versions?: { sdk: string; surface: string }; +} + +/** + * `flows add github:/@#` — resolve the ref to a commit, + * fetch the plugin directory, validate its schema-2 manifest, materialize it + * under `.flows/plugins`, and record the canonical reference in `flows.json` + * plus the provenance in `flows.lock.json`. Runtime composition is a later + * slice: installing records a declaration, it does not enable execution. + */ +export async function addExtensionPlugin(input: string, io: CliIo, options: AddExtensionOptions = {}): Promise<0 | 2> { + try { + // Parse before touching the filesystem or the network: a malformed + // reference is refused offline, with the same code from any directory. + const requested = parsePluginSource(input); + const root = findPluginProject(options.cwd ?? process.cwd()); + if (!root) throw new PluginError('plugin_manifest_invalid', 'flows add requires a project with flows.json.'); + const configPath = join(root, 'flows.json'); + const config = JSON.parse(readFileSync(configPath, 'utf8')); + if (!config || Array.isArray(config) || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every((p: unknown) => typeof p === 'string')))) throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); + const lock = readPluginLock(root); + const source = await resolveGithubSha(requested, options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const ref = canonicalPluginRef(source); + const declared: string[] = config.plugins ?? []; + for (const other of lock.plugins) { + if (other.name === manifest.name && canonicalPluginRef({ ...other.source, ref: other.source.sha }) !== ref) { + throw new PluginError('plugin_manifest_invalid', `Plugin ${manifest.name} is already installed from ${other.source.owner}/${other.source.repo}@${other.source.sha}; remove it before installing another source under the same name.`); + } + } + const { directory, digest } = await materializePlugin(root, manifest.name, fetched.files); + if (digest !== fetched.digest) throw new PluginError('plugin_source_drift', 'Materialized digest differs from the fetched digest.'); + const plugins = declared.includes(ref) ? declared : [...declared, ref]; + const next = lockWithPlugin(lock, plugins, { + name: manifest.name, version: manifest.version, + source: { host: 'github', owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }, + digest, manifestSha256, resolvedAt: (options.now ?? (() => new Date()))().toISOString(), + }); + config.plugins = plugins; + writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); + writePluginLock(root, next); + io.stdout(`Added ${manifest.name}@${manifest.version} (flow-extension) from ${ref}`); + io.stdout(` digest sha256:${digest}`); + io.stdout(` materialized at ${directory}`); + for (const line of describeExtension(manifest)) io.stdout(line); + io.stdout(` recorded in flows.json and ${PLUGIN_LOCK_FILE}; runtime composition is not yet supported (plugin_unsupported at run time)`); + return 0; + } catch (error) { + const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); + io.stderr(`REFUSED [${refusal.code}] ${refusal.message}`); + return 2; + } +} diff --git a/packages/sdk/src/cli/add.ts b/packages/sdk/src/cli/add.ts index 46912359e..1dd82c43f 100644 --- a/packages/sdk/src/cli/add.ts +++ b/packages/sdk/src/cli/add.ts @@ -4,11 +4,18 @@ import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { findPluginProject, probePlugin, readPlugin } from '../plugin-loader.js'; import { PluginError, pluginPackageName } from '../plugin-manifest.js'; +import { isGithubPluginRef } from '../plugin-source.js'; +import { addExtensionPlugin, type AddExtensionOptions } from './add-extension.js'; export async function addPlugin(name: string, io: CliIo, options: { cwd?: string; install?: (packageName: string, root: string) => void; + /** GitHub-sourced flow extensions only; ignored for helper packages. */ + extension?: Omit; } = {}): Promise<0 | 2> { + // A GitHub reference is a schema-2 flow extension; everything else is the + // helper-package path below, which this branch leaves exactly as it was. + if (isGithubPluginRef(name)) return addExtensionPlugin(name, io, { ...options.extension, ...(options.cwd === undefined ? {} : { cwd: options.cwd }) }); try { const packageName = pluginPackageName(name); const root = findPluginProject(options.cwd ?? process.cwd()); diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts new file mode 100644 index 000000000..768c986c2 --- /dev/null +++ b/packages/sdk/src/cli/plugin.ts @@ -0,0 +1,86 @@ +import { readFileSync } from 'node:fs'; +import type { CliIo } from '../cli.js'; +import { fetchGithubPlugin, type FetchLike } from '../plugin-github.js'; +import { lockedPlugins, readPluginLock, type PluginLockEntry } from '../plugin-lock.js'; +import { findPluginProject } from '../plugin-loader.js'; +import { PluginError } from '../plugin-manifest.js'; +import { pluginStoreDirectory, verifyStoredPlugin } from '../plugin-store.js'; + +export type PluginArgs = + | { command: 'plugin'; sub: 'list'; json: boolean } + | { command: 'plugin'; sub: 'verify'; json: boolean; offline: boolean }; + +/** `flows plugin list [--json]` · `flows plugin verify [--json] [--offline]` */ +export function parsePluginArgs(args: readonly string[]): PluginArgs | undefined { + const [sub, ...rest] = args; + if (sub !== 'list' && sub !== 'verify') return undefined; + let json = false; + let offline = false; + for (const arg of rest) { + if (arg === '--json' && !json) json = true; + else if (arg === '--offline' && !offline && sub === 'verify') offline = true; + else return undefined; + } + return sub === 'list' ? { command: 'plugin', sub, json } : { command: 'plugin', sub, json, offline }; +} + +function declaredRefs(root: string): readonly string[] { + let config: { plugins?: unknown }; + try { config = JSON.parse(readFileSync(`${root}/flows.json`, 'utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } + if (config === null || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every(p => typeof p === 'string')))) { + throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be strings.'); + } + return (config.plugins as string[] | undefined) ?? []; +} + +/** + * Cross-check the three records that must agree: `flows.json.plugins` + * (declaration), `flows.lock.json` (provenance), and `.flows/plugins` (bytes). + * With the network, the pinned commit is re-fetched and re-hashed too. + */ +export async function verifyPlugins(root: string, options: { offline: boolean; fetch?: FetchLike }): Promise<{ entry: PluginLockEntry; ref: string; directory: string; remote: 'verified' | 'skipped' }[]> { + const declared = declaredRefs(root).filter(ref => ref.startsWith('github:')); + const locked = lockedPlugins(readPluginLock(root)); + const lockedRefs = locked.map(p => p.ref); + for (const ref of declared) if (!lockedRefs.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.json declares ${ref} but flows.lock.json has no entry for it.`); + for (const ref of lockedRefs) if (!declared.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.lock.json records ${ref} but flows.json does not declare it.`); + if (declared.some((ref, index) => lockedRefs[index] !== ref)) throw new PluginError('plugin_lock_invalid', 'flows.lock.json order differs from flows.json.plugins.'); + const results = []; + for (const { ref, entry, source } of locked) { + const directory = pluginStoreDirectory(root, entry.name, entry.digest); + await verifyStoredPlugin(directory, entry.digest); + let remote: 'verified' | 'skipped' = 'skipped'; + if (!options.offline) { + const fetched = await fetchGithubPlugin(source, options.fetch); + if (fetched.digest !== entry.digest) throw new PluginError('plugin_source_drift', `${ref}: GitHub now serves digest ${fetched.digest}, lockfile has ${entry.digest}.`); + remote = 'verified'; + } + results.push({ entry, ref, directory, remote }); + } + return results; +} + +export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: { cwd?: string; fetch?: FetchLike } = {}): Promise<0 | 2> { + try { + const root = findPluginProject(options.cwd ?? process.cwd()); + if (!root) throw new PluginError('plugin_manifest_invalid', 'flows plugin requires a project with flows.json.'); + if (parsed.sub === 'list') { + const plugins = lockedPlugins(readPluginLock(root)); + if (parsed.json) { io.stdout(JSON.stringify({ plugins: plugins.map(p => ({ ...p.entry, ref: p.ref })) })); return 0; } + if (plugins.length === 0) { io.stdout('No flow-extension plugins installed.'); return 0; } + for (const { entry, ref } of plugins) io.stdout(`${entry.order}. ${entry.name}@${entry.version} ${ref} sha256:${entry.digest}`); + return 0; + } + const results = await verifyPlugins(root, { offline: parsed.offline, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) }); + if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: results.map(r => ({ name: r.entry.name, ref: r.ref, digest: r.entry.digest, remote: r.remote })) })); return 0; } + for (const r of results) io.stdout(`OK ${r.entry.name}@${r.entry.version} ${r.ref} local digest matches lockfile; remote ${r.remote}`); + if (results.length === 0) io.stdout('No flow-extension plugins to verify.'); + return 0; + } catch (error) { + const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); + if (parsed.json) io.stdout(JSON.stringify({ ok: false, code: refusal.code, message: refusal.message })); + io.stderr(`REFUSED [${refusal.code}] ${refusal.message}`); + return 2; + } +} diff --git a/packages/sdk/src/flow-extension-manifest.ts b/packages/sdk/src/flow-extension-manifest.ts new file mode 100644 index 000000000..f18a8a579 --- /dev/null +++ b/packages/sdk/src/flow-extension-manifest.ts @@ -0,0 +1,190 @@ +import { Ajv } from 'ajv'; +import { providerEventTypes } from '@relayflows/surface'; +import { FLOW_HARNESSES } from './flow-requirements.js'; +import { snapshotJsonValue } from './json-value.js'; +import { PluginError, pluginKindOf } from './plugin-manifest.js'; +import { safePath } from './bundle.js'; +import { SHA, parsePluginSource, type PluginSourceInput } from './plugin-source.js'; +import { isVersionRange, parseVersion } from './semver-range.js'; + +/** + * Schema 2, `kind: "flow-extension"`: a plugin whose `entry` default-exports + * `flow()` and contributes handlers, hooks, and gates to a base flow. It is the + * same `flows-plugin.json` file and the same preflight covenant as a helper + * plugin (RFC-0001 decision 13); only the kind decides which validator reads it. + * Helper manifests (`kind` absent) never reach this module. + * + * This slice validates and records the declaration. Runtime composition is + * refused with `plugin_unsupported` (plugin-loader.ts) until the handlers slice. + */ +export interface FlowExtensionTrigger { readonly provider: string; readonly event: string; readonly actions: readonly string[] } +export interface FlowExtensionCompat { + readonly surface: string; + readonly sdk: string; + readonly base: readonly { readonly name: string; readonly version: string }[]; +} +export interface FlowExtensionPermissions { + readonly integrations: readonly string[]; + readonly harnesses: readonly string[]; + readonly mcp: readonly string[]; + /** Declared effect classes, shown for review; not enforced by this runtime (gate 8 / #442). */ + readonly writes: readonly string[]; + readonly budget?: { readonly dollars?: number; readonly wallclock?: string }; +} +export interface FlowExtensionManifest { + readonly schema: 2; + readonly kind: 'flow-extension'; + readonly name: string; + readonly version: string; + readonly description?: string; + /** Authoring copies may omit it; `flows add` records the resolved origin in the lockfile regardless. */ + readonly source?: PluginSourceInput & { readonly sha?: string }; + readonly compat: FlowExtensionCompat; + readonly entry: string; + readonly extends: { readonly handlers: boolean; readonly hooks: readonly string[] }; + readonly triggers: readonly FlowExtensionTrigger[]; + readonly permissions: FlowExtensionPermissions; + readonly preflight: { readonly credentials: readonly string[]; readonly servers: readonly string[] }; + readonly config?: Record; +} + +const TOP_LEVEL = new Set(['schema', 'kind', 'name', 'version', 'description', 'source', 'compat', 'entry', 'extends', 'triggers', 'gates', 'verbs', 'permissions', 'preflight', 'config']); +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; +const PROVIDER = /^[a-z0-9][a-z0-9-]{0,63}$/; +const WRITE_CLASS = /^[a-z0-9-]+(?::[a-z0-9_-]+)+$/; +const WALLCLOCK = /^\d+(?:ms|s|m|h|d)$/; +const MAX_DESCRIPTION = 500; +const MAX_LIST = 64; + +const object = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const invalid = (message: string): never => { throw new PluginError('plugin_manifest_invalid', message); }; + +function stringList(value: unknown, what: string, pattern: RegExp): readonly string[] { + if (!Array.isArray(value) || value.length > MAX_LIST) return invalid(`${what} must be a list of at most ${MAX_LIST} strings.`); + const out: string[] = []; + for (const entry of value) { + if (typeof entry !== 'string' || !pattern.test(entry)) return invalid(`${what} has an invalid entry ${JSON.stringify(entry)}.`); + if (out.includes(entry)) return invalid(`${what} lists ${entry} twice.`); + out.push(entry); + } + return Object.freeze(out); +} + +function compat(value: unknown): FlowExtensionCompat { + if (!object(value) || Object.keys(value).some(k => !['surface', 'sdk', 'base'].includes(k))) return invalid('compat expects surface, sdk, and base.'); + for (const key of ['surface', 'sdk'] as const) { + if (typeof value[key] !== 'string' || !isVersionRange(value[key])) return invalid(`compat.${key} must be a version range (*, x.y.z, ^x.y.z, ~x.y.z, >=x.y.z [ MAX_LIST) return invalid('compat.base must name at least one base flow.'); + const base = value.base.map(entry => { + if (!object(entry) || Object.keys(entry).some(k => !['name', 'version'].includes(k)) + || typeof entry.name !== 'string' || entry.name.trim().length === 0 || entry.name.length > 100 + || typeof entry.version !== 'string' || !isVersionRange(entry.version)) return invalid('compat.base entries are { name, version range }.'); + return Object.freeze({ name: entry.name, version: entry.version }); + }); + if (new Set(base.map(b => b.name)).size !== base.length) return invalid('compat.base names a base flow twice.'); + return Object.freeze({ surface: value.surface as string, sdk: value.sdk as string, base: Object.freeze(base) }); +} + +function triggers(value: unknown): readonly FlowExtensionTrigger[] { + if (!Array.isArray(value) || value.length > MAX_LIST) return invalid('triggers must be a list.'); + const registry = providerEventTypes as Readonly>; + const seen = new Set(); + return Object.freeze(value.map(entry => { + if (!object(entry) || Object.keys(entry).some(k => !['provider', 'event', 'actions'].includes(k)) + || typeof entry.provider !== 'string' || !PROVIDER.test(entry.provider) + || typeof entry.event !== 'string' || !IDENTIFIER.test(entry.event)) return invalid('triggers entries are { provider, event, actions }.'); + const actions = stringList(entry.actions, `triggers ${entry.provider}.${entry.event} actions`, IDENTIFIER); + const known = registry[entry.provider]; + // Fail where ingress would: an event the surface registry cannot lower is + // refused now, not after deployment on the first real delivery. + if (known === undefined) throw new PluginError('plugin_event_unroutable', `Trigger provider ${entry.provider} is not in the surface event registry.`); + const unroutable = (actions.length === 0 ? [entry.event] : actions.map(a => `${entry.event}.${a}`)).filter(type => !known.includes(type)); + if (unroutable.length > 0) throw new PluginError('plugin_event_unroutable', `Trigger ${entry.provider} ${unroutable.join(', ')} is not in the surface event registry.`); + const key = `${entry.provider}:${entry.event}`; + if (seen.has(key)) return invalid(`Trigger ${key} is declared twice.`); + seen.add(key); + return Object.freeze({ provider: entry.provider, event: entry.event, actions }); + })); +} + +function permissions(value: unknown): FlowExtensionPermissions { + if (!object(value) || Object.keys(value).some(k => !['integrations', 'harnesses', 'mcp', 'writes', 'budget'].includes(k))) { + return invalid('permissions expects integrations, harnesses, mcp, writes, and optional budget.'); + } + const harnesses = stringList(value.harnesses, 'permissions.harnesses', /^[a-z]+$/); + for (const harness of harnesses) if (!(FLOW_HARNESSES as readonly string[]).includes(harness)) return invalid(`permissions.harnesses: unknown harness ${harness}.`); + let budget: FlowExtensionPermissions['budget']; + if (value.budget !== undefined) { + if (!object(value.budget) || Object.keys(value.budget).some(k => !['dollars', 'wallclock'].includes(k))) return invalid('permissions.budget expects dollars and/or wallclock.'); + if (value.budget.dollars !== undefined && (typeof value.budget.dollars !== 'number' || !(value.budget.dollars > 0) || !Number.isFinite(value.budget.dollars))) return invalid('permissions.budget.dollars must be a positive number.'); + if (value.budget.wallclock !== undefined && (typeof value.budget.wallclock !== 'string' || !WALLCLOCK.test(value.budget.wallclock))) return invalid('permissions.budget.wallclock must be a duration such as 45m.'); + budget = Object.freeze({ ...(value.budget.dollars === undefined ? {} : { dollars: value.budget.dollars }), ...(value.budget.wallclock === undefined ? {} : { wallclock: value.budget.wallclock }) }); + } + return Object.freeze({ + integrations: stringList(value.integrations, 'permissions.integrations', PROVIDER), + harnesses, + mcp: stringList(value.mcp, 'permissions.mcp', /^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$/), + writes: stringList(value.writes, 'permissions.writes', WRITE_CLASS), + ...(budget === undefined ? {} : { budget }), + }); +} + +function source(value: unknown): FlowExtensionManifest['source'] { + if (!object(value) || Object.keys(value).some(k => !['host', 'owner', 'repo', 'sha', 'path'].includes(k)) + || value.host !== 'github' || typeof value.owner !== 'string' || typeof value.repo !== 'string' + || (value.path !== undefined && typeof value.path !== 'string') + || (value.sha !== undefined && (typeof value.sha !== 'string' || !SHA.test(value.sha)))) { + return invalid('source expects { host: "github", owner, repo, path, sha? }.'); + } + const path = (value.path as string | undefined) ?? ''; + const parsed = parsePluginSource(`github:${value.owner}/${value.repo}@${(value.sha as string | undefined) ?? 'HEAD'}${path === '' ? '' : `#${path}`}`); + return Object.freeze({ ...parsed, ...(value.sha === undefined ? {} : { sha: value.sha as string }) }); +} + +export function validateFlowExtensionManifest(input: unknown): FlowExtensionManifest { + let v: unknown; + try { v = snapshotJsonValue(input, 'plugin manifest'); } + catch { return invalid('Plugin manifest must be JSON data.'); } + if (!object(v)) return invalid('Expected a plugin manifest object.'); + if (pluginKindOf(v) !== 'flow-extension') throw new PluginError('plugin_kind_invalid', 'Expected kind "flow-extension".'); + if (v.schema !== 2) return invalid('A flow-extension manifest is schema 2.'); + const unknown = Object.keys(v).filter(k => !TOP_LEVEL.has(k)); + if (unknown.length > 0) return invalid(`Unknown manifest fields: ${unknown.join(', ')}.`); + if (!Object.hasOwn(v, 'preflight')) throw new PluginError('plugin_preflight_missing', 'Plugin must declare preflight.'); + if (typeof v.name !== 'string' || !NAME.test(v.name) || v.name.startsWith('helper-')) return invalid('name must be lowercase kebab-case and must not start with helper-.'); + if (typeof v.version !== 'string' || parseVersion(v.version) === undefined) return invalid('version must be semver x.y.z.'); + if (v.description !== undefined && (typeof v.description !== 'string' || v.description.length > MAX_DESCRIPTION)) return invalid(`description must be a string of at most ${MAX_DESCRIPTION} characters.`); + if (typeof v.entry !== 'string' || !v.entry.endsWith('.flow.ts') || !safePath(v.entry)) return invalid('entry must be a plugin-relative .flow.ts path.'); + if (!object(v.extends) || Object.keys(v.extends).some(k => !['handlers', 'hooks', 'verbs', 'gates'].includes(k)) || typeof v.extends.handlers !== 'boolean') return invalid('extends expects { handlers: boolean, hooks: [] }.'); + const hooks = stringList(v.extends.hooks ?? [], 'extends.hooks', NAME); + for (const [field, at] of [['verbs', v.extends.verbs], ['gates', v.extends.gates], ['verbs', v.verbs], ['gates', v.gates]] as const) { + if (at !== undefined && (!Array.isArray(at) || at.length > 0)) return invalid(`A flow extension declares no ${field}; ship a helper plugin beside it.`); + } + if (!v.extends.handlers && hooks.length === 0) return invalid('A flow extension must contribute handlers or at least one hook.'); + if (!object(v.preflight) || Object.keys(v.preflight).some(k => !['credentials', 'servers'].includes(k))) return invalid('Preflight requires credentials and servers arrays.'); + const credentials = stringList(v.preflight.credentials, 'preflight.credentials', IDENTIFIER); + const servers = stringList(v.preflight.servers, 'preflight.servers', /^https?:\/\/\S+$/); + for (const server of servers) { + try { if (!['http:', 'https:'].includes(new URL(server).protocol)) return invalid('Servers must be HTTP(S) URLs.'); } + catch { return invalid('Servers must be HTTP(S) URLs.'); } + } + let config: Record | undefined; + if (v.config !== undefined) { + if (!object(v.config)) return invalid('config must be a JSON Schema object.'); + try { new Ajv({ strict: false }).compile(v.config); } catch { return invalid('config is not a valid JSON Schema.'); } + config = v.config; + } + return Object.freeze({ + schema: 2, kind: 'flow-extension', name: v.name, version: v.version, + ...(v.description === undefined ? {} : { description: v.description }), + ...(v.source === undefined ? {} : { source: source(v.source) }), + compat: compat(v.compat), entry: v.entry, + extends: Object.freeze({ handlers: v.extends.handlers, hooks }), + triggers: triggers(v.triggers ?? []), + permissions: permissions(v.permissions), + preflight: Object.freeze({ credentials, servers }), + ...(config === undefined ? {} : { config }), + }); +} diff --git a/packages/sdk/src/plugin-github.ts b/packages/sdk/src/plugin-github.ts new file mode 100644 index 000000000..170eabb0a --- /dev/null +++ b/packages/sdk/src/plugin-github.ts @@ -0,0 +1,98 @@ +import { payloadManifest, safePath, sha256 } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; +import { SHA, type PluginSourceInput, type PluginSourceRef } from './plugin-source.js'; + +/** + * Public, unauthenticated GitHub reads for flow-extension plugins — the same + * posture as Cloud's deploy links: no credential ever travels to GitHub, so a + * private repository simply answers 404 and is reported as unresolved. + * + * Two calls resolve a ref and enumerate a tree; blobs come from + * raw.githubusercontent.com pinned to the resolved commit. Every byte count is + * checked against the tree listing, and the tree is refused if GitHub + * truncated it, if it contains a symlink or submodule under the plugin path, + * or if any file or the whole plugin exceeds the bounds below. + */ +export type FetchLike = (url: string, init: { headers: Record; signal: AbortSignal }) => Promise; + +export const MAX_PLUGIN_FILE_BYTES = 256_000; +export const MAX_PLUGIN_TOTAL_BYTES = 2_000_000; +export const MAX_PLUGIN_FILES = 500; +const TIMEOUT_MS = 15_000; +const API = 'https://api.github.com'; +const RAW = 'https://raw.githubusercontent.com'; + +export interface FetchedPluginFile { readonly path: string; readonly data: Buffer } +export interface FetchedPlugin { + readonly source: PluginSourceRef; + /** Plugin-relative paths, sorted. */ + readonly files: readonly FetchedPluginFile[]; + /** sha256 of `payloadManifest(files)`: the content identity persisted as `@sha256:`. */ + readonly digest: string; +} + +async function request(fetch: FetchLike, url: string, accept: string): Promise { + let response: Response; + try { + response = await fetch(url, { headers: { accept, 'user-agent': 'relayflows-sdk' }, signal: AbortSignal.timeout(TIMEOUT_MS) }); + } catch (error) { + throw new PluginError('plugin_fetch_failed', `GitHub request failed: ${url} (${error instanceof Error ? error.message : String(error)}).`); + } + if (response.status === 404) throw new PluginError('plugin_source_unresolved', `Not found on GitHub (or not public): ${url}.`); + if (!response.ok) throw new PluginError('plugin_fetch_failed', `GitHub answered ${response.status} for ${url}.`); + return response; +} + +/** Branch, tag, or commit → the commit sha it names right now; a sha input is verified to exist. */ +export async function resolveGithubSha(source: PluginSourceInput, fetch: FetchLike = globalThis.fetch): Promise { + const url = `${API}/repos/${source.owner}/${source.repo}/commits/${encodeURIComponent(source.ref)}`; + const sha = (await (await request(fetch, url, 'application/vnd.github.sha')).text()).trim(); + if (!SHA.test(sha)) throw new PluginError('plugin_fetch_failed', `GitHub returned a malformed commit sha for ${source.ref}.`); + if (SHA.test(source.ref) && sha !== source.ref) throw new PluginError('plugin_source_unresolved', `Commit ${source.ref} resolved to ${sha}.`); + return Object.freeze({ ...source, sha }); +} + +interface TreeEntry { path: string; mode: string; type: string; size?: number } + +async function listTree(source: PluginSourceRef, fetch: FetchLike): Promise { + const url = `${API}/repos/${source.owner}/${source.repo}/git/trees/${source.sha}?recursive=1`; + let body: unknown; + try { body = await (await request(fetch, url, 'application/vnd.github+json')).json(); } + catch { throw new PluginError('plugin_fetch_failed', 'GitHub tree listing is not JSON.'); } + const tree = typeof body === 'object' && body !== null ? (body as { tree?: unknown; truncated?: unknown }) : {}; + if (tree.truncated === true) throw new PluginError('plugin_fetch_failed', 'GitHub truncated the tree listing; the repository is too large to enumerate safely.'); + if (!Array.isArray(tree.tree)) throw new PluginError('plugin_fetch_failed', 'GitHub tree listing has no entries.'); + return tree.tree.filter((e): e is TreeEntry => typeof e === 'object' && e !== null + && typeof (e as TreeEntry).path === 'string' && typeof (e as TreeEntry).mode === 'string' && typeof (e as TreeEntry).type === 'string'); +} + +/** Enumerate and download the plugin directory at the pinned commit, bounded and verified. */ +export async function fetchGithubPlugin(source: PluginSourceRef, fetch: FetchLike = globalThis.fetch): Promise { + const prefix = source.path === '' ? '' : `${source.path}/`; + const entries = (await listTree(source, fetch)).filter(e => e.path.startsWith(prefix)); + if (source.path !== '' && entries.length === 0) throw new PluginError('plugin_source_unresolved', `${source.path} does not exist at ${source.sha}.`); + const blobs: { path: string; size: number }[] = []; + let total = 0; + for (const entry of entries) { + const relative = entry.path.slice(prefix.length); + if (entry.type === 'tree') continue; + if (entry.type === 'commit') throw new PluginError('plugin_path_invalid', `${entry.path} is a submodule; plugins must be plain files.`); + if (entry.mode === '120000') throw new PluginError('plugin_path_invalid', `${entry.path} is a symlink; plugins must be plain files.`); + if (entry.type !== 'blob' || !safePath(relative)) throw new PluginError('plugin_path_invalid', `${entry.path} is not a plain repository file.`); + if (!Number.isSafeInteger(entry.size) || entry.size! < 0) throw new PluginError('plugin_fetch_failed', `${entry.path} has no size in the tree listing.`); + if (entry.size! > MAX_PLUGIN_FILE_BYTES) throw new PluginError('plugin_too_large', `${entry.path} is ${entry.size} bytes; the limit is ${MAX_PLUGIN_FILE_BYTES}.`); + total += entry.size!; + if (total > MAX_PLUGIN_TOTAL_BYTES) throw new PluginError('plugin_too_large', `Plugin exceeds ${MAX_PLUGIN_TOTAL_BYTES} bytes in total.`); + blobs.push({ path: relative, size: entry.size! }); + if (blobs.length > MAX_PLUGIN_FILES) throw new PluginError('plugin_too_large', `Plugin has more than ${MAX_PLUGIN_FILES} files.`); + } + if (!blobs.some(b => b.path === 'flows-plugin.json')) throw new PluginError('plugin_manifest_missing', `${source.owner}/${source.repo}@${source.sha}${prefix === '' ? '' : `#${source.path}`} has no flows-plugin.json.`); + const files: FetchedPluginFile[] = []; + for (const blob of blobs.sort((a, b) => a.path < b.path ? -1 : 1)) { + const url = `${RAW}/${source.owner}/${source.repo}/${source.sha}/${prefix}${blob.path}`; + const data = Buffer.from(await (await request(fetch, url, 'application/octet-stream')).arrayBuffer()); + if (data.length !== blob.size) throw new PluginError('plugin_source_drift', `${blob.path}: fetched ${data.length} bytes, tree lists ${blob.size}.`); + files.push(Object.freeze({ path: blob.path, data })); + } + return Object.freeze({ source, files: Object.freeze(files), digest: sha256(payloadManifest(files)) }); +} diff --git a/packages/sdk/src/plugin-loader.ts b/packages/sdk/src/plugin-loader.ts index 8911590ea..e4352b0b5 100644 --- a/packages/sdk/src/plugin-loader.ts +++ b/packages/sdk/src/plugin-loader.ts @@ -5,6 +5,7 @@ import { Ajv } from 'ajv'; import type { Step } from '@relayflows/surface'; import { snapshotJsonValue } from './json-value.js'; import { assertSupportedPlugin, PluginError, pluginPackageName, validatePluginManifest, type PluginManifest, type PluginVerb } from './plugin-manifest.js'; +import { isGithubPluginRef } from './plugin-source.js'; export interface LoadedPlugin { readonly directory: string; readonly manifest: PluginManifest } export function findPluginProject(start: string): string | undefined { @@ -47,6 +48,11 @@ export async function loadPlugins(start: string): Promise typeof p === 'string')))) { throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be package names.'); } + // Flow extensions (github:… entries) are declared and locked by `flows add` + // but not yet composed at run time. Refuse before any helper loads rather + // than silently running the base flow without them. + const extension = (config.plugins as string[] | undefined)?.find(isGithubPluginRef); + if (extension !== undefined) throw new PluginError('plugin_unsupported', `${extension} is a flow-extension plugin; runtime composition of flow extensions is not supported by this release.`); const scope = join(root, 'node_modules/@flows'); const names = new Set((config.plugins as string[] | undefined)?.map(pluginPackageName)); if (existsSync(scope)) for (const name of readdirSync(scope).sort()) { diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts new file mode 100644 index 000000000..ee6f00184 --- /dev/null +++ b/packages/sdk/src/plugin-lock.ts @@ -0,0 +1,99 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PluginError } from './plugin-manifest.js'; +import { SHA, canonicalPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; + +/** + * `flows.lock.json` — the project's plugin provenance. `flows.json.plugins` + * says *what* is declared; the lockfile says exactly which bytes that meant: + * the commit, the content digest, the manifest hash, and the order the + * operator declared. Version 2 because the sealed bundle's `lockfile.json` + * is version 1 and the two will converge on this shape when bundles carry + * plugins (RFC-0001 decision 14). + */ +export const PLUGIN_LOCK_FILE = 'flows.lock.json'; +export const PLUGIN_LOCK_VERSION = 2; + +export interface PluginLockEntry { + readonly name: string; + readonly kind: 'flow-extension'; + readonly version: string; + readonly source: { readonly host: 'github'; readonly owner: string; readonly repo: string; readonly sha: string; readonly path: string }; + /** sha256 of the payload manifest — the `@sha256:` in `.flows/plugins`. */ + readonly digest: string; + /** sha256 of the `flows-plugin.json` bytes as installed. */ + readonly manifestSha256: string; + /** 1-based position in `flows.json.plugins`; the deterministic composition order. */ + readonly order: number; + readonly resolvedAt: string; +} +export interface PluginLock { readonly version: 2; readonly plugins: readonly PluginLockEntry[] } + +const HEX64 = /^[0-9a-f]{64}$/; +const object = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const invalid = (message: string): never => { throw new PluginError('plugin_lock_invalid', `${PLUGIN_LOCK_FILE}: ${message}`); }; + +export function parsePluginLock(input: unknown): PluginLock { + if (!object(input) || input.version !== PLUGIN_LOCK_VERSION || !Array.isArray(input.plugins) + || Object.keys(input).some(k => !['version', 'plugins'].includes(k))) return invalid(`expected { version: ${PLUGIN_LOCK_VERSION}, plugins: [] }.`); + const names = new Set(); + const plugins = input.plugins.map((entry, index) => { + if (!object(entry) || Object.keys(entry).sort().join(',') !== 'digest,kind,manifestSha256,name,order,resolvedAt,source,version' + || entry.kind !== 'flow-extension' || typeof entry.name !== 'string' || typeof entry.version !== 'string' + || typeof entry.digest !== 'string' || !HEX64.test(entry.digest) + || typeof entry.manifestSha256 !== 'string' || !HEX64.test(entry.manifestSha256) + || entry.order !== index + 1 || typeof entry.resolvedAt !== 'string' || Number.isNaN(Date.parse(entry.resolvedAt)) + || !object(entry.source) || entry.source.host !== 'github' || typeof entry.source.owner !== 'string' + || typeof entry.source.repo !== 'string' || typeof entry.source.sha !== 'string' || !SHA.test(entry.source.sha) + || typeof entry.source.path !== 'string') return invalid(`plugins[${index}] is malformed.`); + if (names.has(entry.name)) return invalid(`plugin ${entry.name} is listed twice.`); + names.add(entry.name); + const source = parseCanonicalPluginRef(canonicalPluginRef({ host: 'github', owner: entry.source.owner, repo: entry.source.repo, ref: entry.source.sha, sha: entry.source.sha, path: entry.source.path })); + return Object.freeze({ + name: entry.name, kind: 'flow-extension' as const, version: entry.version, + source: Object.freeze({ host: 'github' as const, owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }), + digest: entry.digest, manifestSha256: entry.manifestSha256, order: entry.order, resolvedAt: entry.resolvedAt, + }); + }); + return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze(plugins) }); +} + +/** Absent file → empty lock; unreadable or malformed → refusal. */ +export function readPluginLock(root: string): PluginLock { + const path = join(root, PLUGIN_LOCK_FILE); + if (!existsSync(path)) return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze([]) }); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(path, 'utf8')); } + catch { return invalid('not valid JSON.'); } + return parsePluginLock(parsed); +} + +export function writePluginLock(root: string, lock: PluginLock): void { + writeFileSync(join(root, PLUGIN_LOCK_FILE), `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); +} + +/** + * Rebuild the entry list in `flows.json.plugins` order: `order` is derived from + * the declaration list, never stored independently, so the two cannot disagree. + */ +export function lockWithPlugin( + lock: PluginLock, declared: readonly string[], + entry: Omit, +): PluginLock { + const byRef = new Map(lock.plugins.map(p => [canonicalPluginRef({ ...p.source, ref: p.source.sha }), p])); + byRef.set(canonicalPluginRef({ ...entry.source, ref: entry.source.sha }), { ...entry, kind: 'flow-extension', order: 0 }); + const plugins = declared.filter(ref => ref.startsWith('github:')).map((ref, index) => { + const found = byRef.get(ref); + if (found === undefined) return invalid(`flows.json declares ${ref} but the lockfile has no entry for it; run flows add ${ref}.`); + return Object.freeze({ ...found, order: index + 1 }); + }); + return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze(plugins) }); +} + +/** Lock entries paired with their declared reference, in declaration order. */ +export function lockedPlugins(lock: PluginLock): readonly { ref: string; entry: PluginLockEntry; source: PluginSourceRef }[] { + return lock.plugins.map(entry => { + const source = { ...entry.source, ref: entry.source.sha }; + return { ref: canonicalPluginRef(source), entry, source }; + }); +} diff --git a/packages/sdk/src/plugin-manifest.ts b/packages/sdk/src/plugin-manifest.ts index ba133444a..fb4791d79 100644 --- a/packages/sdk/src/plugin-manifest.ts +++ b/packages/sdk/src/plugin-manifest.ts @@ -5,6 +5,10 @@ export const PLUGIN_FAILURE_KINDS = [ 'plugin_unknown', 'plugin_unlisted', 'plugin_install_failed', 'plugin_manifest_missing', 'plugin_manifest_invalid', 'plugin_preflight_missing', 'plugin_verb_unknown_primitive', 'plugin_unsupported', 'plugin_credential_missing', 'plugin_server_unreachable', + // schema 2 flow extensions (see flow-extension-manifest.ts, plugin-source.ts, plugin-github.ts) + 'plugin_kind_invalid', 'plugin_incompatible', 'plugin_event_unroutable', 'plugin_source_invalid', + 'plugin_source_unresolved', 'plugin_fetch_failed', 'plugin_path_invalid', 'plugin_too_large', + 'plugin_source_drift', 'plugin_lock_invalid', ] as const; export type PluginFailureKind = typeof PLUGIN_FAILURE_KINDS[number]; export class PluginError extends Error { @@ -16,6 +20,15 @@ export interface PluginVerb { lowersTo: 'run' | 'llm' | 'agent' | 'effect' | 'wait'; args: Record; } +/** The kind a `flows-plugin.json` declares; absent means the schema-1 helper plugin. */ +export type PluginKind = 'helper' | 'flow-extension'; +export function pluginKindOf(input: unknown): PluginKind { + const kind = typeof input === 'object' && input !== null && !Array.isArray(input) ? (input as { kind?: unknown }).kind : undefined; + if (kind === undefined || kind === 'helper') return 'helper'; + if (kind === 'flow-extension') return 'flow-extension'; + throw new PluginError('plugin_kind_invalid', `Unknown plugin kind ${JSON.stringify(kind)}; expected "helper" or "flow-extension".`); +} +/** A schema-1 helper plugin: verbs that lower to kernel primitives. */ export interface PluginManifest { name: string; version: string; @@ -41,6 +54,10 @@ export function validatePluginManifest(input: unknown, packageName?: string): Pl catch { throw new PluginError('plugin_manifest_invalid', 'Plugin manifest must be JSON data.'); } const invalid = (message: string): never => { throw new PluginError('plugin_manifest_invalid', message); }; if (!object(v)) return invalid('Expected a plugin manifest object.'); + if (pluginKindOf(v) !== 'helper') { + throw new PluginError('plugin_kind_invalid', 'A flow-extension manifest installs from a GitHub reference (flows add github:/@#), not as a helper package.'); + } + if (v.schema !== undefined && v.schema !== 1) return invalid('Helper plugin manifests are schema 1.'); if (!Object.hasOwn(v, 'preflight')) throw new PluginError('plugin_preflight_missing', 'Plugin must declare preflight.'); if (typeof v.name !== 'string' || typeof v.version !== 'string' || !v.version.trim()) return invalid('Plugin name and version are required.'); const resolved = pluginPackageName(v.name); diff --git a/packages/sdk/src/plugin-source.ts b/packages/sdk/src/plugin-source.ts new file mode 100644 index 000000000..a9923dbe6 --- /dev/null +++ b/packages/sdk/src/plugin-source.ts @@ -0,0 +1,87 @@ +import { safePath } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; + +/** + * Where a flow-extension plugin comes from: a public GitHub repository, pinned + * to a commit. A branch or tag is accepted as *input* only; what gets written + * to `flows.json` and the lockfile is always the canonical 40-hex form, + * `github:/@#`, so a later reader can never resolve + * to different bytes than the installer saw. + */ +export interface PluginSourceInput { + readonly host: 'github'; + readonly owner: string; + readonly repo: string; + /** Branch, tag, or commit as typed. */ + readonly ref: string; + /** Directory inside the repository holding `flows-plugin.json`; `''` is the root. */ + readonly path: string; +} +export interface PluginSourceRef extends PluginSourceInput { + /** Exactly 40 lowercase hex characters. */ + readonly sha: string; +} + +const OWNER = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; +const REPO = /^[A-Za-z0-9_.-]{1,100}$/; +const REF = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$/; +export const SHA = /^[0-9a-f]{40}$/; + +function invalid(message: string): never { + throw new PluginError('plugin_source_invalid', message); +} + +export function isGithubPluginRef(value: string): boolean { + return value.startsWith('github:') || /^https:\/\/github\.com\//.test(value); +} + +/** + * Accepts `github:/@#`, + * `https://github.com///tree//` (or `/blob/`), and + * `github:/@` for a root-level plugin. + * + * In the URL form the ref is the single segment after `tree/`: GitHub itself + * disambiguates `tree/feat/x/dir` against the repository's refs, which an + * offline parser cannot. A ref containing `/` must use the `github:` form, + * where `@ref#path` is unambiguous. + */ +export function parsePluginSource(input: string): PluginSourceInput { + if (input.length > 2048) return invalid('Plugin source is too long.'); + let owner: string | undefined, repo: string | undefined, ref: string | undefined, path = ''; + if (input.startsWith('github:')) { + const m = /^github:([^/@#]+)\/([^/@#]+)@([^#]+)(?:#(.*))?$/.exec(input); + if (!m) return invalid('Expected github:/@[#].'); + [, owner, repo, ref] = m; + path = m[4] ?? ''; + } else if (/^https:\/\/github\.com\//.test(input)) { + let url: URL; + try { url = new URL(input); } catch { return invalid('Plugin source is not a valid URL.'); } + if (url.username || url.password || url.search || url.hash) return invalid('Plugin URL must not carry credentials, a query, or a fragment.'); + const m = /^\/([^/]+)\/([^/]+)\/(?:tree|blob)\/([^/]+)(?:\/(.*))?$/.exec(url.pathname); + if (!m) return invalid('Expected https://github.com///tree//.'); + [, owner, repo, ref] = m.map(part => part === undefined ? part : decodeURIComponent(part)); + path = m[4] === undefined ? '' : decodeURIComponent(m[4]); + } else return invalid('Expected a github: reference or a https://github.com/ URL.'); + if (owner === undefined || !OWNER.test(owner)) return invalid('Invalid GitHub owner.'); + if (repo === undefined || !REPO.test(repo) || repo === '.' || repo === '..') return invalid('Invalid GitHub repository name.'); + if (repo.endsWith('.git')) repo = repo.slice(0, -4); + if (ref === undefined || !REF.test(ref) || ref.includes('..') || ref.endsWith('/') || ref.endsWith('.lock')) return invalid('Invalid git ref.'); + path = path.replace(/\/+$/, ''); + if (path !== '' && !safePath(path)) return invalid('Plugin path must be repository-relative without traversal.'); + if (path.split('/').some(part => part === 'flows-plugin.json')) return invalid('Plugin path names the directory holding flows-plugin.json, not the file.'); + return Object.freeze({ host: 'github', owner, repo, ref, path }); +} + +/** The one spelling that is ever persisted. */ +export function canonicalPluginRef(source: PluginSourceRef): string { + return `github:${source.owner}/${source.repo}@${source.sha}${source.path === '' ? '' : `#${source.path}`}`; +} + +/** Parses a persisted reference; refuses anything but the canonical sha form. */ +export function parseCanonicalPluginRef(value: string): PluginSourceRef { + const parsed = parsePluginSource(value); + if (!value.startsWith('github:') || !SHA.test(parsed.ref)) { + throw new PluginError('plugin_source_invalid', `Persisted plugin reference must be github:/@[#], got ${value}.`); + } + return Object.freeze({ ...parsed, sha: parsed.ref }); +} diff --git a/packages/sdk/src/plugin-store.ts b/packages/sdk/src/plugin-store.ts new file mode 100644 index 000000000..61b5275f8 --- /dev/null +++ b/packages/sdk/src/plugin-store.ts @@ -0,0 +1,90 @@ +import { lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { payloadManifest, safePath, sha256 } from './bundle.js'; +import { PluginError } from './plugin-manifest.js'; + +/** + * Where a flow-extension plugin's bytes live inside a project: + * `/.flows/plugins/@sha256:/`, content-addressed like + * the bundle cache and never `node_modules`. `manifest.json` in that directory + * is the payload manifest whose sha256 is the digest, so a directory can be + * re-verified without the network — and, because the lockfile records the + * same digest, drift between what was installed and what is on disk is a + * refusal, not a surprise. + */ +export const PLUGIN_STORE = '.flows/plugins'; + +export function pluginStoreDirectory(root: string, name: string, digest: string): string { + return join(resolve(root), PLUGIN_STORE, `${name}@sha256:${digest}`); +} + +export interface StoredPluginFile { readonly path: string; readonly data: Uint8Array } + +/** Write the files atomically; an existing directory is verified instead of overwritten. */ +export async function materializePlugin(root: string, name: string, files: readonly StoredPluginFile[]): Promise<{ directory: string; digest: string }> { + const manifest = payloadManifest(files); + const digest = sha256(manifest); + const directory = pluginStoreDirectory(root, name, digest); + let exists = false; + try { await lstat(directory); exists = true; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + if (exists) { await verifyStoredPlugin(directory, digest); return { directory, digest }; } + const parent = dirname(directory); + await mkdir(parent, { recursive: true }); + const staging = await mkdtemp(join(parent, '.install-')); + try { + for (const file of files) { + if (!safePath(file.path) || file.path === 'manifest.json') throw new PluginError('plugin_path_invalid', `${file.path}: invalid plugin path.`); + await mkdir(dirname(join(staging, file.path)), { recursive: true }); + await writeFile(join(staging, file.path), file.data, { mode: 0o644 }); + } + await writeFile(join(staging, 'manifest.json'), manifest); + try { await rename(staging, directory); } + catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; + await verifyStoredPlugin(directory, digest); + } + } finally { await rm(staging, { recursive: true, force: true }); } + return { directory, digest }; +} + +async function regularFile(root: string, path: string): Promise { + const parts = path.split('/'); + for (let i = 1; i <= parts.length; i++) { + const stat = await lstat(join(root, ...parts.slice(0, i))); + if (i === parts.length ? !stat.isFile() : !stat.isDirectory()) throw new PluginError('plugin_source_drift', `${path}: expected a regular file, without symlinks.`); + } + return readFile(join(root, path)); +} + +/** Re-hash a materialized plugin and compare with the digest the lockfile recorded. */ +export async function verifyStoredPlugin(directory: string, expectedDigest: string): Promise { + const drift = (message: string): never => { throw new PluginError('plugin_source_drift', `${directory}: ${message}`); }; + let raw: string; + try { + if (!(await lstat(directory)).isDirectory()) return drift('not a directory'); + raw = (await regularFile(directory, 'manifest.json')).toString('utf8'); + } catch (error) { return drift(error instanceof PluginError ? error.message : 'manifest.json is missing'); } + if (sha256(raw) !== expectedDigest) return drift('manifest.json digest differs from the lockfile'); + let entries: { path: string; sha256: string; bytes: number }[]; + try { entries = JSON.parse(raw); if (!Array.isArray(entries)) throw new Error(); } + catch { return drift('manifest.json is not a manifest'); } + const paths = new Set(); + for (const entry of entries) { + if (typeof entry?.path !== 'string' || !safePath(entry.path)) return drift('manifest.json lists an invalid path'); + let data: Buffer; + try { data = await regularFile(directory, entry.path); } + catch (error) { return drift(error instanceof PluginError ? error.message : `${entry.path} is missing`); } + if (data.length !== entry.bytes || sha256(data) !== entry.sha256) return drift(`${entry.path} changed since installation`); + paths.add(entry.path); + } + await rejectExtras(directory, '', new Set([...paths, 'manifest.json']), drift); +} + +async function rejectExtras(root: string, prefix: string, paths: Set, drift: (m: string) => never): Promise { + for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { + const path = prefix + entry.name; + if (entry.isDirectory() && [...paths].some(file => file.startsWith(`${path}/`))) await rejectExtras(root, `${path}/`, paths, drift); + else if (!entry.isFile() || !paths.has(path)) drift(`${path}: unlisted file or unsupported file type`); + } +} diff --git a/packages/sdk/src/semver-range.ts b/packages/sdk/src/semver-range.ts new file mode 100644 index 000000000..b09532499 --- /dev/null +++ b/packages/sdk/src/semver-range.ts @@ -0,0 +1,52 @@ +/** + * The small semver subset plugin `compat` ranges may use. Deliberately not a + * dependency on `semver`: a plugin's compatibility claim has to be checkable by + * every reader — Cloud, the CLI, a catalog — from one short, obvious rule. + * + * Accepted: `*`, `x.y.z`, `^x.y.z`, `~x.y.z`, `>=x.y.z`, `>=x.y.z =)?\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?: <\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)?)$/; + +interface Parsed { readonly triple: readonly [number, number, number]; readonly pre?: string } + +export function parseVersion(value: string): Parsed | undefined { + const m = VERSION.exec(value); + if (!m) return undefined; + const triple = [Number(m[1]), Number(m[2]), Number(m[3])] as const; + if (triple.some(n => !Number.isSafeInteger(n))) return undefined; + return m[4] === undefined ? { triple } : { triple, pre: m[4] }; +} + +export function isVersionRange(value: string): boolean { + return RANGE.test(value); +} + +function compare(a: Parsed, b: Parsed): number { + for (let i = 0; i < 3; i++) if (a.triple[i] !== b.triple[i]) return a.triple[i]! - b.triple[i]!; + if (a.pre === b.pre) return 0; + if (a.pre === undefined) return 1; + if (b.pre === undefined) return -1; + return a.pre < b.pre ? -1 : 1; +} + +/** Whether `version` satisfies `range`; false for malformed input rather than a throw. */ +export function satisfiesRange(version: string, range: string): boolean { + const v = parseVersion(version); + if (v === undefined || !isVersionRange(range)) return false; + if (range === '*') return true; + const [lowerText, upperText] = range.split(' ') as [string, string | undefined]; + const operator = /^[\^~]|^>=/.exec(lowerText)?.[0] ?? ''; + const lower = parseVersion(lowerText.slice(operator.length))!; + if (compare(v, lower) < 0) return false; + if (operator === '') return compare(v, lower) === 0; + let upper: Parsed | undefined; + if (upperText !== undefined) upper = parseVersion(upperText.slice(1)); + else if (operator === '^') { + const [major, minor] = lower.triple; + upper = major > 0 ? { triple: [major + 1, 0, 0] } : minor > 0 ? { triple: [0, minor + 1, 0] } : { triple: [0, 0, lower.triple[2] + 1] }; + } else if (operator === '~') upper = { triple: [lower.triple[0], lower.triple[1] + 1, 0] }; + return upper === undefined || compare(v, upper) < 0; +} diff --git a/packages/sdk/tests/fake-github.ts b/packages/sdk/tests/fake-github.ts new file mode 100644 index 000000000..19dde3098 --- /dev/null +++ b/packages/sdk/tests/fake-github.ts @@ -0,0 +1,70 @@ +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import type { FetchLike } from '../src/plugin-github.js'; + +/** + * An in-memory stand-in for the three public GitHub reads the SDK performs: + * commit resolution, recursive tree listing, and raw blob download. Tests + * shape repositories directly (symlinks, submodules, oversize files, byte + * drift, truncated trees) so every refusal is exercised offline. + */ +export interface FakeEntry { path: string; data?: Buffer; mode?: string; type?: string; size?: number } +export interface FakeCommit { entries: FakeEntry[]; truncated?: boolean } +export interface FakeRepo { refs: Record; commits: Record } +export interface FakeGithub { fetch: FetchLike; calls: string[]; repos: Record } + +export const SHA_A = 'a'.repeat(40); +export const SHA_B = 'b'.repeat(40); + +/** Every file under `directory`, mounted at `mountPath/` inside the fake repository. */ +export function entriesFromDirectory(directory: string, mountPath: string): FakeEntry[] { + const entries: FakeEntry[] = []; + const walk = (dir: string): void => { + for (const name of readdirSync(dir).sort()) { + const full = join(dir, name); + if (statSync(full).isDirectory()) { walk(full); continue; } + const path = `${mountPath === '' ? '' : `${mountPath}/`}${relative(directory, full).split('\\').join('/')}`; + entries.push({ path, data: readFileSync(full) }); + } + }; + walk(directory); + return entries; +} + +export function fakeGithub(repos: Record): FakeGithub { + const calls: string[] = []; + const fetch: FetchLike = async (url) => { + calls.push(url); + const u = new URL(url); + const respond = (status: number, body: BodyInit | null = null, type = 'text/plain'): Response => new Response(body, { status, headers: { 'content-type': type } }); + if (u.host === 'api.github.com') { + let m = /^\/repos\/([^/]+)\/([^/]+)\/commits\/(.+)$/.exec(u.pathname); + if (m) { + const repo = repos[`${m[1]}/${m[2]}`]; + const ref = decodeURIComponent(m[3]!); + const sha = repo?.refs[ref] ?? (repo?.commits[ref] ? ref : undefined); + return sha === undefined ? respond(404) : respond(200, sha); + } + m = /^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([0-9a-f]{40})$/.exec(u.pathname); + if (m) { + const commit = repos[`${m[1]}/${m[2]}`]?.commits[m[3]!]; + if (!commit) return respond(404); + const dirs = new Set(); + for (const e of commit.entries) { const parts = e.path.split('/'); for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join('/')); } + const tree = [ + ...[...dirs].map(path => ({ path, mode: '040000', type: 'tree', sha: SHA_B })), + ...commit.entries.map(e => ({ path: e.path, mode: e.mode ?? '100644', type: e.type ?? 'blob', sha: SHA_B, size: e.size ?? e.data?.length ?? 0 })), + ]; + return respond(200, JSON.stringify({ sha: m[3], tree, truncated: commit.truncated ?? false }), 'application/json'); + } + return respond(404); + } + if (u.host === 'raw.githubusercontent.com') { + const m = /^\/([^/]+)\/([^/]+)\/([0-9a-f]{40})\/(.+)$/.exec(u.pathname); + const entry = m && repos[`${m[1]}/${m[2]}`]?.commits[m[3]!]?.entries.find(e => e.path === decodeURIComponent(m[4]!)); + return entry?.data === undefined ? respond(404) : respond(200, new Uint8Array(entry.data), 'application/octet-stream'); + } + return respond(404); + }; + return { fetch, calls, repos }; +} diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts new file mode 100644 index 000000000..ca252b64c --- /dev/null +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -0,0 +1,299 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { addPlugin } from '../src/cli/add.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; +import { runPluginCommand, verifyPlugins } from '../src/cli/plugin.js'; +import { runCli } from '../src/cli.js'; +import { validateFlowExtensionManifest } from '../src/flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha } from '../src/plugin-github.js'; +import { loadPlugins } from '../src/plugin-loader.js'; +import { parsePluginLock, readPluginLock } from '../src/plugin-lock.js'; +import { validatePluginManifest } from '../src/plugin-manifest.js'; +import { canonicalPluginRef, parseCanonicalPluginRef, parsePluginSource } from '../src/plugin-source.js'; +import { pluginStoreDirectory } from '../src/plugin-store.js'; +import { satisfiesRange } from '../src/semver-range.js'; +import { SHA_A, SHA_B, entriesFromDirectory, fakeGithub, type FakeEntry } from './fake-github.js'; + +const fixtureRoot = resolve('../../testdata/plugins'); +const babysitter = entriesFromDirectory(join(fixtureRoot, 'extension-babysitter'), 'examples/babysitter'); +const manifestJson = JSON.parse(readFileSync(join(fixtureRoot, 'extension-babysitter/flows-plugin.json'), 'utf8')); +const REF = `github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`; +const versions = { sdk: '2.0.22', surface: '2.0.22' }; +const now = () => new Date('2026-09-20T12:00:00Z'); +const dirs: string[] = []; +afterEach(() => { dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); vi.unstubAllEnvs(); }); + +function github(entries: FakeEntry[] = babysitter, extra: Partial<{ truncated: boolean }> = {}) { + return fakeGithub({ 'AgentWorkforce/flows': { refs: { 'feat/babysitter-v2': SHA_A, 'v0.1.0': SHA_A, main: SHA_B }, commits: { [SHA_A]: { entries, ...extra }, [SHA_B]: { entries: [] } } } }); +} +function project(config: unknown = {}) { + const cwd = mkdtempSync(join(tmpdir(), 'plugin-ext-')); dirs.push(cwd); + writeFileSync(join(cwd, 'flows.json'), JSON.stringify(config)); + const messages: string[] = []; + const io = { stdout: (s: string) => messages.push(s), stderr: (s: string) => messages.push(s) }; + return { cwd, io, messages, text: () => messages.join('\n') }; +} +function withManifest(patch: (m: Record) => unknown): FakeEntry[] { + return babysitter.map(e => e.path.endsWith('flows-plugin.json') ? { ...e, data: Buffer.from(JSON.stringify(patch(structuredClone(manifestJson)))) } : e); +} + +describe('plugin source references', () => { + it('parses the three accepted spellings to one shape', () => { + const expected = { host: 'github', owner: 'AgentWorkforce', repo: 'flows', ref: 'feat/babysitter-v2', path: 'examples/babysitter' }; + expect(parsePluginSource('github:AgentWorkforce/flows@feat/babysitter-v2#examples/babysitter')).toEqual(expected); + expect(parsePluginSource('https://github.com/AgentWorkforce/flows/tree/v0.1.0/examples/babysitter')).toEqual({ ...expected, ref: 'v0.1.0' }); + expect(parsePluginSource('https://github.com/AgentWorkforce/flows.git/blob/v0.1.0/examples/babysitter/')).toEqual({ ...expected, ref: 'v0.1.0' }); + expect(parsePluginSource('github:o/r@main')).toMatchObject({ path: '' }); + // The URL form cannot tell a slash in the ref from a path segment; that is + // what the github: form is for. Documented, not guessed. + expect(parsePluginSource('https://github.com/AgentWorkforce/flows/tree/feat/babysitter-v2/examples/babysitter')).toMatchObject({ ref: 'feat', path: 'babysitter-v2/examples/babysitter' }); + }); + it.each([ + 'github:o/r@main#../etc', 'github:o/r@main#a/../b', 'github:o/r@main#/abs', 'github:o/r@../x', 'github:o/r@main#a\\b', + 'https://github.com/o/r/tree/main/x?token=1', 'https://user:pw@github.com/o/r/tree/main/x', 'https://gitlab.com/o/r/tree/main/x', + 'github:o/r', 'github:o/r@main#examples/flows-plugin.json', 'github:-bad/r@main', 'github:o/r@main.lock', + ])('refuses %s', input => { + expect(() => parsePluginSource(input)).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + }); + it('persists only the canonical sha form', () => { + expect(canonicalPluginRef({ host: 'github', owner: 'o', repo: 'r', ref: SHA_A, sha: SHA_A, path: '' })).toBe(`github:o/r@${SHA_A}`); + expect(parseCanonicalPluginRef(REF).sha).toBe(SHA_A); + expect(() => parseCanonicalPluginRef('github:o/r@main#x')).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + expect(() => parseCanonicalPluginRef(`https://github.com/o/r/tree/${SHA_A}/x`)).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); + }); +}); + +describe('semver ranges', () => { + it.each([ + ['2.0.22', '^2.0.22', true], ['2.9.0', '^2.0.22', true], ['3.0.0', '^2.0.22', false], ['2.0.21', '^2.0.22', false], + ['2.0.30', '~2.0.22', true], ['2.1.0', '~2.0.22', false], ['5.0.0', '>=2.0.0', true], ['2.5.0', '>=2.0.0 <2.5.0', false], + ['2.0.22', '2.0.22', true], ['2.0.23', '2.0.22', false], ['0.0.9', '*', true], ['0.1.5', '^0.1.0', true], ['0.2.0', '^0.1.0', false], + ['2.0.22', 'latest', false], ['x', '*', false], + ])('%s satisfies %s → %s', (version, range, ok) => { expect(satisfiesRange(version, range)).toBe(ok); }); +}); + +describe('flows add ', () => { + it('resolves a branch to its commit, materializes, and records flows.json plus the lockfile', async () => { + const gh = github(); const p = project({ cli: 'claude' }); + expect(await addPlugin('github:AgentWorkforce/flows@feat/babysitter-v2#examples/babysitter', p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + const config = JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')); + expect(config).toEqual({ cli: 'claude', plugins: [REF] }); + const lock = readPluginLock(p.cwd); + expect(lock.plugins).toHaveLength(1); + const [entry] = lock.plugins; + expect(entry).toMatchObject({ name: 'babysitter', kind: 'flow-extension', version: '0.1.0', order: 1, resolvedAt: '2026-09-20T12:00:00.000Z', source: { host: 'github', owner: 'AgentWorkforce', repo: 'flows', sha: SHA_A, path: 'examples/babysitter' } }); + expect(entry!.digest).toMatch(/^[0-9a-f]{64}$/); + const store = pluginStoreDirectory(p.cwd, 'babysitter', entry!.digest); + expect(existsSync(join(store, 'flows-plugin.json'))).toBe(true); + expect(existsSync(join(store, 'babysitter.flow.ts'))).toBe(true); + expect(existsSync(join(store, 'manifest.json'))).toBe(true); + expect(p.text()).toContain(`Added babysitter@0.1.0 (flow-extension) from ${REF}`); + expect(p.text()).toContain('events: github pull_request[opened,synchronize,reopened,closed]; github pull_request_review[submitted,dismissed]; github check_run[completed]; github issue_comment[created]'); + expect(p.text()).toContain('writes (declared, unenforced): github:pull_request:comment'); + expect(p.text()).toContain('runtime composition is not yet supported'); + expect(gh.calls.some(url => url.includes('/commits/feat%2Fbabysitter-v2'))).toBe(true); + // A tag naming the same commit is a no-op re-add: no duplicate declaration, same lock entry. + expect(await addPlugin('https://github.com/AgentWorkforce/flows/tree/v0.1.0/examples/babysitter', p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + expect(readPluginLock(p.cwd)).toEqual(lock); + // And a sha input is accepted as-is. + expect(await addPlugin(REF, p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); + }); + it('keeps the digest stable across identical installs and distinct across content changes', async () => { + const gh = github(); const a = project(); const b = project(); + await addExtensionPlugin(REF, a.io, { cwd: a.cwd, fetch: gh.fetch, now, versions }); + await addExtensionPlugin(REF, b.io, { cwd: b.cwd, fetch: gh.fetch, now, versions }); + expect(readPluginLock(a.cwd)).toEqual(readPluginLock(b.cwd)); + const changed = github(withManifest(m => ({ ...m, description: 'changed' }))); + const c = project(); + await addExtensionPlugin(REF, c.io, { cwd: c.cwd, fetch: changed.fetch, now, versions }); + expect(readPluginLock(c.cwd).plugins[0]!.digest).not.toBe(readPluginLock(a.cwd).plugins[0]!.digest); + }); + it.each([ + ['github:AgentWorkforce/flows@nope#examples/babysitter', 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@main#examples/babysitter', 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@feat/babysitter-v2#examples', 'plugin_manifest_missing'], + [`github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`, 'plugin_source_unresolved'], + ['github:AgentWorkforce/flows@feat/babysitter-v2#../x', 'plugin_source_invalid'], + ])('refuses %s with %s and writes nothing', async (input, code) => { + const gh = github(); const p = project(); + expect(await addPlugin(input, p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(2); + expect(p.text()).toContain(`REFUSED [${code}]`); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8'))).toEqual({}); + expect(existsSync(join(p.cwd, 'flows.lock.json'))).toBe(false); + expect(existsSync(join(p.cwd, '.flows'))).toBe(false); + }); + it('refuses a plugin whose compat excludes this runtime', async () => { + const gh = github(); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions: { sdk: '2.0.22', surface: '3.0.0' } })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_incompatible] babysitter requires surface ^2.0.22; this runtime has 3.0.0.'); + }); + it('refuses a manifest whose declared source is not where it was fetched from', async () => { + const gh = github(withManifest(m => ({ ...m, source: { host: 'github', owner: 'someone', repo: 'else', path: 'examples/babysitter' } }))); + const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_drift]'); + }); + it('refuses a second source under an already-installed name', async () => { + const gh = github(); const p = project(); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + gh.repos['other/fork'] = { refs: { main: SHA_A }, commits: { [SHA_A]: { entries: babysitter } } }; + expect(await addExtensionPlugin(`github:other/fork@main#examples/babysitter`, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(2); + expect(p.text()).toContain('already installed from AgentWorkforce/flows'); + }); +}); + +describe('bounded, verified fetches', () => { + const source = { host: 'github' as const, owner: 'AgentWorkforce', repo: 'flows', ref: SHA_A, sha: SHA_A, path: 'examples/babysitter' }; + const plus = (entry: FakeEntry) => [...babysitter, entry]; + it.each([ + ['a symlink', plus({ path: 'examples/babysitter/link', data: Buffer.from('x'), mode: '120000' }), 'plugin_path_invalid'], + ['a submodule', plus({ path: 'examples/babysitter/vendor', type: 'commit', mode: '160000' }), 'plugin_path_invalid'], + ['a traversal path', plus({ path: 'examples/babysitter/a/../b', data: Buffer.from('x') }), 'plugin_path_invalid'], + ['a backslash path', plus({ path: 'examples/babysitter/a\\b', data: Buffer.from('x') }), 'plugin_path_invalid'], + ['an oversize file', plus({ path: 'examples/babysitter/big.bin', data: Buffer.alloc(256_001) }), 'plugin_too_large'], + ['a byte-count mismatch', plus({ path: 'examples/babysitter/drift.txt', data: Buffer.from('abc'), size: 2 }), 'plugin_source_drift'], + ])('refuses %s', async (_, entries, code) => { + await expect(fetchGithubPlugin(source, github(entries).fetch)).rejects.toMatchObject({ code }); + }); + it('refuses a plugin that exceeds the total byte budget', async () => { + const entries = [...babysitter, ...Array.from({ length: 9 }, (_, i) => ({ path: `examples/babysitter/part-${i}.bin`, data: Buffer.alloc(250_000) }))]; + await expect(fetchGithubPlugin(source, github(entries).fetch)).rejects.toMatchObject({ code: 'plugin_too_large' }); + }); + it('refuses a truncated tree listing rather than installing a partial plugin', async () => { + await expect(fetchGithubPlugin(source, github(babysitter, { truncated: true }).fetch)).rejects.toMatchObject({ code: 'plugin_fetch_failed', message: expect.stringContaining('truncated') }); + }); + it('reports a private or missing repository as unresolved, never as a transport error', async () => { + await expect(resolveGithubSha({ ...source, ref: 'main', owner: 'private', repo: 'repo' }, github().fetch)).rejects.toMatchObject({ code: 'plugin_source_unresolved' }); + const failing = async () => { throw new Error('ECONNRESET'); }; + await expect(resolveGithubSha(source, failing)).rejects.toMatchObject({ code: 'plugin_fetch_failed' }); + }); +}); + +describe('schema-2 manifest validation', () => { + it('accepts the worked Babysitter manifest and freezes it', () => { + const m = validateFlowExtensionManifest(manifestJson); + expect(m).toMatchObject({ schema: 2, kind: 'flow-extension', name: 'babysitter', entry: 'babysitter.flow.ts', extends: { handlers: true, hooks: ['merge-gate'] } }); + expect(m.triggers).toHaveLength(4); + expect(Object.isFrozen(m) && Object.isFrozen(m.permissions) && Object.isFrozen(m.triggers)).toBe(true); + }); + it.each([ + ['an event the surface registry cannot lower', (m: Record) => ({ ...m, triggers: [{ provider: 'github', event: 'pull_request', actions: ['ready_for_review'] }] }), 'plugin_event_unroutable'], + ['labeled/unlabeled, which the registry lacks', (m: Record) => ({ ...m, triggers: [{ provider: 'github', event: 'pull_request', actions: ['labeled', 'unlabeled'] }] }), 'plugin_event_unroutable'], + ['an unknown provider', (m: Record) => ({ ...m, triggers: [{ provider: 'nope', event: 'x', actions: [] }] }), 'plugin_event_unroutable'], + ['an unknown kind', (m: Record) => ({ ...m, kind: 'banana' }), 'plugin_kind_invalid'], + ['schema 1 with the extension kind', (m: Record) => ({ ...m, schema: 1 }), 'plugin_manifest_invalid'], + ['verbs on a flow extension', (m: Record) => ({ ...m, verbs: [{ namespace: 'x', method: 'y', lowersTo: 'effect', args: {} }] }), 'plugin_manifest_invalid'], + ['an unknown top-level field', (m: Record) => ({ ...m, extra: 1 }), 'plugin_manifest_invalid'], + ['a helper- name', (m: Record) => ({ ...m, name: 'helper-x' }), 'plugin_manifest_invalid'], + ['a non-semver version', (m: Record) => ({ ...m, version: 'v1' }), 'plugin_manifest_invalid'], + ['a malformed compat range', (m: Record) => ({ ...m, compat: { ...(m.compat as object), surface: 'latest' } }), 'plugin_manifest_invalid'], + ['a traversal entry', (m: Record) => ({ ...m, entry: '../x.flow.ts' }), 'plugin_manifest_invalid'], + ['neither handlers nor hooks', (m: Record) => ({ ...m, extends: { handlers: false, hooks: [] } }), 'plugin_manifest_invalid'], + ['an unknown harness', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), harnesses: ['cursor'] } }), 'plugin_manifest_invalid'], + ['a malformed write class', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), writes: ['github'] } }), 'plugin_manifest_invalid'], + ['a non-positive budget', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 0 } } }), 'plugin_manifest_invalid'], + ['a config that is not a JSON Schema', (m: Record) => ({ ...m, config: { type: 'not-a-type' } }), 'plugin_manifest_invalid'], + ['a missing preflight', (m: Record) => { const { preflight, ...rest } = m; void preflight; return rest; }, 'plugin_preflight_missing'], + ])('refuses %s', (_, patch, code) => { + expect(() => validateFlowExtensionManifest(patch(structuredClone(manifestJson)))).toThrow(expect.objectContaining({ code })); + }); + it('keeps the helper validator helper-only and the extension validator extension-only', () => { + expect(() => validatePluginManifest(manifestJson)).toThrow(expect.objectContaining({ code: 'plugin_kind_invalid' })); + const helper = JSON.parse(readFileSync(join(fixtureRoot, 'helper-datadog/flows-plugin.json'), 'utf8')); + expect(validatePluginManifest(helper).name).toBe('helper-datadog'); + expect(validatePluginManifest({ ...helper, kind: 'helper', schema: 1 }).name).toBe('helper-datadog'); + expect(() => validatePluginManifest({ ...helper, schema: 2 })).toThrow(expect.objectContaining({ code: 'plugin_manifest_invalid' })); + expect(() => validateFlowExtensionManifest(helper)).toThrow(expect.objectContaining({ code: 'plugin_kind_invalid' })); + }); +}); + +describe('flows plugin list / verify', () => { + async function installed() { + const gh = github(); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + p.messages.length = 0; + return { gh, p, digest: readPluginLock(p.cwd).plugins[0]!.digest }; + } + it('lists in composition order and verifies locally and remotely', async () => { + const { gh, p, digest } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'list', json: false }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toBe(`1. babysitter@0.1.0 ${REF} sha256:${digest}`); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'list', json: true }, p.io, { cwd: p.cwd })).toBe(0); + expect(JSON.parse(p.text()).plugins[0]).toMatchObject({ name: 'babysitter', ref: REF, digest }); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: true }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toContain('remote skipped'); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: true, offline: false }, p.io, { cwd: p.cwd, fetch: gh.fetch })).toBe(0); + expect(JSON.parse(p.text())).toEqual({ ok: true, plugins: [{ name: 'babysitter', ref: REF, digest, remote: 'verified' }] }); + }); + it('refuses when GitHub serves different bytes at the pinned commit', async () => { + const { p } = await installed(); + const drifted = github(withManifest(m => ({ ...m, description: 'rewritten history' }))); + await expect(verifyPlugins(p.cwd, { offline: false, fetch: drifted.fetch })).rejects.toMatchObject({ code: 'plugin_source_drift', message: expect.stringContaining('GitHub now serves digest') }); + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: false }, p.io, { cwd: p.cwd, fetch: drifted.fetch })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_drift]'); + }); + it.each([ + ['an edited file', (store: string) => writeFileSync(join(store, 'babysitter.flow.ts'), '// tampered')], + ['a deleted file', (store: string) => rmSync(join(store, 'babysitter.flow.ts'))], + ['an added file', (store: string) => writeFileSync(join(store, 'extra.ts'), '')], + ['a symlink in place of a file', (store: string) => { rmSync(join(store, 'README.md')); symlinkSync('/etc/hostname', join(store, 'README.md')); }], + ['an edited manifest', (store: string) => writeFileSync(join(store, 'manifest.json'), '[]')], + ])('refuses the local store after %s', async (_, tamper) => { + const { p, digest } = await installed(); + tamper(pluginStoreDirectory(p.cwd, 'babysitter', digest)); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_source_drift' }); + }); + it('refuses when flows.json and the lockfile disagree', async () => { + const { p } = await installed(); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ plugins: [] })); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_lock_invalid', message: expect.stringContaining('does not declare') }); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ plugins: [REF, `github:o/r@${SHA_B}`] })); + await expect(verifyPlugins(p.cwd, { offline: true })).rejects.toMatchObject({ code: 'plugin_lock_invalid', message: expect.stringContaining('no entry') }); + }); + it('refuses a malformed lockfile', () => { + expect(() => parsePluginLock({ version: 1, plugins: [] })).toThrow(expect.objectContaining({ code: 'plugin_lock_invalid' })); + const entry = { name: 'x', kind: 'flow-extension', version: '1.0.0', source: { host: 'github', owner: 'o', repo: 'r', sha: SHA_A, path: '' }, digest: 'f'.repeat(64), manifestSha256: 'f'.repeat(64), order: 2, resolvedAt: '2026-09-20T00:00:00Z' }; + expect(() => parsePluginLock({ version: 2, plugins: [entry] })).toThrow(expect.objectContaining({ code: 'plugin_lock_invalid', message: expect.stringContaining('plugins[0]') })); + expect(parsePluginLock({ version: 2, plugins: [{ ...entry, order: 1 }] }).plugins[0]!.order).toBe(1); + expect(() => parsePluginLock({ version: 2, plugins: [{ ...entry, order: 1 }, { ...entry, order: 2 }] })).toThrow(expect.objectContaining({ message: expect.stringContaining('twice') })); + }); +}); + +describe('legacy helper plugins are untouched', () => { + it('never contacts GitHub for a helper name and leaves the helper path to npm', async () => { + const gh = github(); const p = project(); + const install = vi.fn(() => { throw { stderr: 'offline' }; }); + expect(await addPlugin('helper-datadog', p.io, { cwd: p.cwd, install, extension: { fetch: gh.fetch } })).toBe(2); + expect(install).toHaveBeenCalledWith('@flows/helper-datadog', p.cwd); + expect(gh.calls).toEqual([]); + expect(p.text()).toContain('plugin_install_failed'); + }); + it('refuses runtime composition of a declared flow extension, fail closed, before loading helpers', async () => { + const gh = github(); const p = project(); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + await expect(loadPlugins(p.cwd)).rejects.toMatchObject({ code: 'plugin_unsupported', message: expect.stringContaining(REF) }); + expect(await loadPlugins(project().cwd)).toEqual([]); + }); + it('dispatches through the CLI: add refuses a bad reference offline, plugin list and verify run', async () => { + const p = project(); + expect(await runCli(['add', 'github:o/r@main#../x'], p.io)).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_source_invalid]'); + const previous = process.cwd(); + process.chdir(p.cwd); + try { + p.messages.length = 0; + expect(await runCli(['plugin', 'list'], p.io)).toBe(0); + expect(p.text()).toBe('No flow-extension plugins installed.'); + p.messages.length = 0; + expect(await runCli(['plugin', 'verify', '--offline', '--json'], p.io)).toBe(0); + expect(JSON.parse(p.text())).toEqual({ ok: true, plugins: [] }); + expect(await runCli(['plugin', 'nope'], p.io)).toBe(2); + } finally { process.chdir(previous); } + }); +}); diff --git a/packages/sdk/tests/preflight.test.ts b/packages/sdk/tests/preflight.test.ts index 6c5596f9c..a8fc0e68b 100644 --- a/packages/sdk/tests/preflight.test.ts +++ b/packages/sdk/tests/preflight.test.ts @@ -2,6 +2,8 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { addPlugin } from '../src/cli/add.js'; +import { runPluginCommand } from '../src/cli/plugin.js'; +import { SHA_A, fakeGithub, type FakeEntry } from './fake-github.js'; import type { PreflightFailureKind } from '../src/failure-kinds.js'; import { preflightHelpers } from '../src/preflight.js'; import { describe, expect, it } from 'vitest'; @@ -623,6 +625,49 @@ describe('preflight: CLI resolution and refusal predicates', () => { } } finally { rmSync(root, { recursive: true, force: true }); } } + // Flow-extension refusals (schema 2) exercise `flows add ` and + // `flows plugin verify` against an offline fake GitHub — the same public + // boundary a user hits, never a synthesized diagnostic. + { + const manifest = { + schema: 2, kind: 'flow-extension', name: 'ext', version: '0.1.0', entry: 'ext.flow.ts', + compat: { surface: '*', sdk: '*', base: [{ name: 'base', version: '*' }] }, + extends: { handlers: true, hooks: [] }, triggers: [], + permissions: { integrations: [], harnesses: [], mcp: [], writes: [] }, + preflight: { credentials: [], servers: [] }, + }; + const files = (m: unknown): FakeEntry[] => [ + { path: 'ext/flows-plugin.json', data: Buffer.from(JSON.stringify(m)) }, + { path: 'ext/ext.flow.ts', data: Buffer.from('export default 1;') }, + ]; + const repo = (entries: FakeEntry[]) => fakeGithub({ 'o/r': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries } } } }).fetch; + const extensionCases: { ref: string; fetch: import('../src/plugin-github.js').FetchLike }[] = [ + { ref: 'github:o/r@main#../x', fetch: repo(files(manifest)) }, + { ref: 'github:o/r@nope#ext', fetch: repo(files(manifest)) }, + { ref: 'github:o/r@main#ext', fetch: async () => { throw new Error('offline'); } }, + { ref: 'github:o/r@main#ext', fetch: repo([...files(manifest), { path: 'ext/link', data: Buffer.from('x'), mode: '120000' }]) }, + { ref: 'github:o/r@main#ext', fetch: repo([...files(manifest), { path: 'ext/big', data: Buffer.alloc(256_001) }]) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, kind: 'banana' })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, triggers: [{ provider: 'github', event: 'pull_request', actions: ['ready_for_review'] }] })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, compat: { ...manifest.compat, surface: '^1.0.0' } })) }, + { ref: 'github:o/r@main#ext', fetch: repo(files({ ...manifest, source: { host: 'github', owner: 'someone', repo: 'else', path: 'ext' } })) }, + ]; + for (const { ref, fetch } of extensionCases) { + const root = mkdtempSync(join(tmpdir(), 'plugin-extension-taxonomy-')); + try { + writeFileSync(join(root, 'flows.json'), '{}'); + const io = { stdout() {}, stderr(line: string) { refusalKinds.push(line.match(/\[([^\]]+)\]/)![1] as PreflightFailureKind); } }; + expect(await addPlugin(ref, io, { cwd: root, extension: { fetch, versions: { sdk: '2.0.22', surface: '2.0.22' } } })).toBe(2); + } finally { rmSync(root, { recursive: true, force: true }); } + } + // `plugin_lock_invalid`: a declaration with no lockfile entry behind it. + const root = mkdtempSync(join(tmpdir(), 'plugin-lock-taxonomy-')); + try { + writeFileSync(join(root, 'flows.json'), JSON.stringify({ plugins: [`github:o/r@${SHA_A}#ext`] })); + const io = { stdout() {}, stderr(line: string) { refusalKinds.push(line.match(/\[([^\]]+)\]/)![1] as PreflightFailureKind); } }; + expect(await runPluginCommand({ command: 'plugin', sub: 'verify', json: false, offline: true }, io, { cwd: root })).toBe(2); + } finally { rmSync(root, { recursive: true, force: true }); } + } // `plugin_unlisted` fires when a @flows/helper-* package is present in // node_modules but is missing from the declared plugins list — the loader // refuses to auto-load undeclared packages (plugin-loader.ts). Exercise it diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 3af09a9d3..97caf8b3d 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -52,6 +52,7 @@ const RUN_ID = '01JABCDEFGHJKMNPQRSTVWXYZ0'; */ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: ParsedArgs['command'] }[] = [ { verb: 'add', argv: ['add', 'my-helper'], variant: 'add' }, + { verb: 'add', argv: ['add', 'github:AgentWorkforce/flows@main#examples/babysitter'], variant: 'add' }, { verb: 'answer', argv: ['answer', RUN_ID, 'human-1', 'yes'], variant: 'answer' }, { verb: 'answer', @@ -164,6 +165,10 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa '--poll-interval-ms', '1000', 'spec.json'], variant: 'tick', }, + { verb: 'plugin', argv: ['plugin', 'list'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'list', '--json'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'verify'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'verify', '--json', '--offline'], variant: 'plugin' }, { verb: 'undeploy', argv: ['undeploy', 'dep_123'], variant: 'undeploy' }, { verb: 'undeploy', argv: ['undeploy', '--json', 'dep_123'], variant: 'undeploy' }, { verb: 'unschedule', argv: ['unschedule', 'sched_123'], variant: 'unschedule' }, diff --git a/testdata/plugins/extension-babysitter/README.md b/testdata/plugins/extension-babysitter/README.md new file mode 100644 index 000000000..85c11cbef --- /dev/null +++ b/testdata/plugins/extension-babysitter/README.md @@ -0,0 +1,14 @@ +# extension-babysitter (offline fixture) + +The worked schema-2 `kind: "flow-extension"` manifest for Babysitter on +Software Garden, served to the SDK tests by a fake GitHub (see +`packages/sdk/tests/plugin-extension.test.ts`). It is a fixture, not an +installable example: the entry is a stub, and runtime composition of flow +extensions is refused with `plugin_unsupported` in this release. + +Babysitter's own subscription contract (branch `feat/babysitter-v2`) names +eleven GitHub subscriptions. Three of them — `pull_request.ready_for_review`, +`pull_request.labeled`, `pull_request.unlabeled` — are not in the surface +event registry (`providerEventTypes`), so a manifest declaring them is refused +with `plugin_event_unroutable`; this fixture lists only the eight the registry +can lower. Extending the registry is a separate change and is not claimed here. diff --git a/testdata/plugins/extension-babysitter/babysitter.flow.ts b/testdata/plugins/extension-babysitter/babysitter.flow.ts new file mode 100644 index 000000000..6b7d4bb2b --- /dev/null +++ b/testdata/plugins/extension-babysitter/babysitter.flow.ts @@ -0,0 +1,5 @@ +// Offline fixture entry. The real Babysitter body lives on the babysitter +// branch of AgentWorkforce/flows; this stub exists so the manifest's `entry` +// resolves inside the fixture. Runtime composition is not implemented yet. +import { flow } from '@relayflows/surface'; +export default flow('babysitter', async (f) => { f.done('declined'); }); diff --git a/testdata/plugins/extension-babysitter/flows-plugin.json b/testdata/plugins/extension-babysitter/flows-plugin.json new file mode 100644 index 000000000..c730c925d --- /dev/null +++ b/testdata/plugins/extension-babysitter/flows-plugin.json @@ -0,0 +1,42 @@ +{ + "schema": 2, + "kind": "flow-extension", + "name": "babysitter", + "version": "0.1.0", + "description": "Live-state PR babysitter: parallel review lenses, deterministic reconciliation, exact-head merge gate. Worked schema-2 example; the real flow lives on the babysitter branch of AgentWorkforce/flows.", + "compat": { + "surface": "^2.0.22", + "sdk": "^2.0.22", + "base": [{ "name": "software-factory", "version": "*" }] + }, + "entry": "babysitter.flow.ts", + "extends": { "handlers": true, "hooks": ["merge-gate"] }, + "triggers": [ + { "provider": "github", "event": "pull_request", "actions": ["opened", "synchronize", "reopened", "closed"] }, + { "provider": "github", "event": "pull_request_review", "actions": ["submitted", "dismissed"] }, + { "provider": "github", "event": "check_run", "actions": ["completed"] }, + { "provider": "github", "event": "issue_comment", "actions": ["created"] } + ], + "permissions": { + "integrations": ["github"], + "harnesses": ["claude"], + "mcp": [], + "writes": ["github:pull_request:comment"], + "budget": { "dollars": 8, "wallclock": "45m" } + }, + "preflight": { "credentials": [], "servers": ["https://api.github.com"] }, + "config": { + "type": "object", + "properties": { + "testCommand": { "type": "string" }, + "botLogin": { "type": "string" }, + "approvers": { "type": "array", "items": { "type": "string" } }, + "organizations": { "type": "array", "items": { "type": "string" } }, + "merge": { "type": "boolean", "default": false }, + "skipLabels": { "type": "array", "items": { "type": "string" }, "default": ["no-agent-relay-review"] }, + "requiredChecks": { "type": "array", "items": { "type": "string" } } + }, + "required": ["testCommand", "approvers"], + "additionalProperties": false + } +} From 89be148585f8ce2339133251e1072387703c8d64 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 15:25:59 -0700 Subject: [PATCH 02/17] feat(sdk): compose schema-2 flow extensions onto the base flow at load time (P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `loadAuthoredFlow` now verifies and composes the project's flow extensions (flow-extension-loader.ts). The order of operations is the security argument and is fixed: flows.json.plugins and flows.lock.json must agree; the materialized store is re-hashed against the lock's digest and the manifest bytes against its manifest hash before anything under .flows/plugins is read as code; the manifest is validated and its compat checked against the runtime and the base flow (the surface header has no version field, so only "*" is satisfiable; a budget ceiling above the base is plugin_incompatible); only then is the entry imported, and its handlers are checked against the manifest's declared triggers — an entry cannot subscribe to more than it declared. Handlers are appended after the base's own, in lockfile order; the base definition object is untouched, and the composed definition is served for the root handle only. Fail closed, refused rather than ignored: hooks, an entry `use:` header, schedule triggers, gates (plugin_unsupported); a generic webhook handler or an undeclared subscription (plugin_manifest_invalid); a foreign surface runtime (plugin_incompatible). Cloud deploy and hosted runs refuse a project with extensions (unsupported_source) — the deploy body carries one source file and would silently drop them. The helper loader now treats github: entries as not-helpers rather than refusing the whole project; the authored loader owns them. `flows check` prints one EXTENSION line per composed extension and keeps plugin refusal codes in its report; the composed trigger set passes preflightProviderTriggers. The Babysitter fixture entry now carries the handler surface (one .on() per declared subscription, eight the registry can lower) over a body that only declines; extends.hooks is empty because hooks are not composed. Handler bodies still execute nowhere (#301): what composition changes today is the declared trigger set. Tests: flow-extension-compose.test.ts (18) — composition order incl. two extensions forward and reverse, base untouched, graph nodes, flows check report and CLI output, extensions: 'none', tampered store refused before import, lock disagreement, runtime/base/budget incompatibility, hooks, undeclared/schedule/generic-webhook/no-handler/use:-header/forged entries, and a registry-unroutable action that an entry cannot smuggle past the manifest. Full SDK vitest: 2507 passed, the one remaining failure is the pre-existing Bun 1.4.0 pin in authored-node-runtime.test.ts on a 1.4.2 host. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 19498b5b-4a5c-4096-a978-84d7082bd5a4 --- docs/SURFACE.md | 25 ++- packages/sdk/src/authored-flow-loader.ts | 49 ++++- packages/sdk/src/cli.ts | 4 + packages/sdk/src/cli/add-extension.ts | 16 +- packages/sdk/src/cli/check-triggers.ts | 11 + packages/sdk/src/cli/check.ts | 12 ++ packages/sdk/src/cli/plugin.ts | 21 +- packages/sdk/src/cloud-deploy.ts | 6 + packages/sdk/src/cloud-run.ts | 4 + packages/sdk/src/flow-extension-compat.ts | 38 ++++ packages/sdk/src/flow-extension-loader.ts | 153 ++++++++++++++ packages/sdk/src/plugin-loader.ts | 10 +- packages/sdk/src/plugin-lock.ts | 36 +++- .../sdk/tests/flow-extension-compose.test.ts | 190 ++++++++++++++++++ packages/sdk/tests/plugin-extension.test.ts | 16 +- .../plugins/extension-babysitter/README.md | 11 +- .../extension-babysitter/babysitter.flow.ts | 22 +- .../extension-babysitter/flows-plugin.json | 121 +++++++++-- 18 files changed, 665 insertions(+), 80 deletions(-) create mode 100644 packages/sdk/src/flow-extension-compat.ts create mode 100644 packages/sdk/src/flow-extension-loader.ts create mode 100644 packages/sdk/tests/flow-extension-compose.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 3c064cd59..0e780f837 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -598,10 +598,27 @@ composition order. `flows plugin verify` re-hashes the store against the lock and, unless `--offline`, re-fetches the pinned commit; any difference is `plugin_source_drift`, exit 2. -Installing records a declaration; it does not enable execution. A project that -declares a flow extension is refused at run time with `plugin_unsupported` -until the handlers/hooks slice lands, so a base flow never silently runs -without the extension it was told it had. +**Composition.** `loadAuthoredFlow` (the path under `flows check`, `flows run`, +and the authored root) composes the project's extensions onto the base flow +(`packages/sdk/src/flow-extension-loader.ts`), in this fixed order: the +declaration and the lockfile must agree; the store is re-hashed against the +lock's digest and the manifest bytes against its manifest hash — nothing under +`.flows/plugins` is read as code before that passes; the manifest is validated +and its `compat` checked against the runtime and the base flow (the base has no +version field yet, so only `*` is satisfiable; a budget ceiling above the base +is `plugin_incompatible`); only then is the entry imported, its handlers +checked against the manifest's declared triggers (an entry cannot subscribe to +more than it declared), and appended **after** the base's own handlers in +lockfile order. Nothing replaces, reorders, or widens a base handler, and the +base's definition object is untouched. `flows check` prints one `EXTENSION` +line per composed extension. Not composed by this release, and refused with +`plugin_unsupported` rather than ignored: hooks, an entry `use:` header, +schedule triggers, and gates; a generic `webhook(...)` handler is refused as +undeclared. Cloud deploy and hosted runs refuse a project with extensions +(`unsupported_source`) because the deploy body carries one source file and +would silently lose them. Handler bodies still execute nowhere (#301); what +composition changes today is the declared trigger set that `flows check`, +requirements, and future dispatch read. ## 4. Build: the immutable bundle diff --git a/packages/sdk/src/authored-flow-loader.ts b/packages/sdk/src/authored-flow-loader.ts index 04dd45aca..22b6f8d28 100644 --- a/packages/sdk/src/authored-flow-loader.ts +++ b/packages/sdk/src/authored-flow-loader.ts @@ -9,6 +9,9 @@ import { type AuthoredFlowDefinition, type FlowHandle, } from './authored-flow.js'; +import { canonicalize } from './canonical.js'; +import { composeDefinition, loadFlowExtensions, type LoadedFlowExtension } from './flow-extension-loader.js'; +import type { RuntimeVersions } from './flow-extension-compat.js'; export class AuthoredFlowLoadError extends Error { constructor(message: string, readonly kind: 'invalid_spec' | 'use_not_found' | 'use_invalid' | 'use_cycle' = 'invalid_spec') { @@ -40,8 +43,20 @@ export interface LoadedAuthoredFlow { readonly getDefinition: GetFlowDefinition; /** Exact Surface package/runtime that owns the handle's WeakMap identity. */ readonly surfaceAuthority: SurfaceModuleAuthority; - /** Dependency-first load order, each canonical absolute path appearing once. */ + /** + * Dependency-first load order, each canonical absolute path appearing once. + * Flow-extension entries follow the root, so a graph of one node still + * means "one self-contained source" to the hosted paths that require it. + */ readonly graph: readonly LoadedAuthoredFlowNode[]; + /** Schema-2 flow extensions composed onto the root, in lock order; empty when the project declares none. */ + readonly extensions: readonly LoadedFlowExtension[]; +} + +export interface LoadAuthoredFlowOptions { + /** `compose` (default) verifies and appends the project's flow extensions; `none` loads the root alone. */ + readonly extensions?: 'compose' | 'none'; + readonly versions?: RuntimeVersions; } export interface LoadedAuthoredFlowNode { @@ -53,7 +68,7 @@ export interface LoadedAuthoredFlowNode { } /** Import and validate a direct-run module without executing its authored body. */ -export async function loadAuthoredFlow(path: string): Promise { +export async function loadAuthoredFlow(path: string, options: LoadAuthoredFlowOptions = {}): Promise { const loaded = new Map(); const visiting = new Set(); async function visit(sourcePath: string, isRoot = false): Promise { @@ -98,8 +113,34 @@ export async function loadAuthoredFlow(path: string): Promise canonicalize(a) === canonicalize(b), + ...(options.versions === undefined ? {} : { versions: options.versions }), + }); + if (extensions.length === 0) { + return Object.freeze({ sourcePath: root.path, handle: root.handle, getDefinition: root.getDefinition, + surfaceAuthority: root.surfaceAuthority, graph: Object.freeze(graph), extensions }); + } + const composed = composeDefinition(baseDefinition, extensions); + const getDefinition: GetFlowDefinition = (handle: FlowHandle) => + (handle === root.handle ? composed : root.getDefinition(handle)) as AuthoredFlowDefinition; + for (const extension of extensions) { + graph.push(Object.freeze({ path: extension.entryPath, handle: extension.handle, getDefinition: root.getDefinition, + surfaceAuthority: root.surfaceAuthority, use: Object.freeze([]) })); + } + return Object.freeze({ sourcePath: root.path, handle: root.handle, getDefinition, + surfaceAuthority: root.surfaceAuthority, graph: Object.freeze(graph), extensions }); } async function importAuthoredFlow(path: string): Promise> { diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index a11ea5dad..24ac1fe6f 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -353,6 +353,7 @@ async function checkAuthoredFlowComposed(path: string): Promise<{ report: CheckR report: { ...mcp.report, ...(triggers?.report.schedules === undefined ? {} : { schedules: triggers.report.schedules }), + ...(triggers?.report.extensions === undefined ? {} : { extensions: triggers.report.extensions }), // The authored definition sees helper flags, body use and `cli:` // declarations; the compiled view underneath knows only its steps. ...(triggers?.report.requirements === undefined ? {} : { requirements: triggers.report.requirements }), @@ -944,6 +945,9 @@ function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { : `local: flows tick start --schedule-id ${schedule.scheduleId} --interval-ms ${schedule.intervalMs} --epoch-ms ${schedule.epochMs}`; io.stdout(`SCHEDULE handler ${schedule.handler} ${declared} -> flows.tick schedule_id ${schedule.scheduleId} [${local}]`); } + for (const extension of report.extensions ?? []) { + io.stdout(`EXTENSION ${extension.name}@${extension.version} ${extension.ref} sha256:${extension.digest} -> ${extension.handlers} handler(s) composed after the base flow`); + } for (const resolution of report.resolutions) { const config = resolution.source === 'project' && report.projectConfigPath !== undefined ? ` (${report.projectConfigPath})` diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts index f21a542ce..c8e4bc9ae 100644 --- a/packages/sdk/src/cli/add-extension.ts +++ b/packages/sdk/src/cli/add-extension.ts @@ -2,6 +2,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { sha256 } from '../bundle.js'; +import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha, type FetchLike, type FetchedPlugin } from '../plugin-github.js'; import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, writePluginLock } from '../plugin-lock.js'; @@ -9,19 +10,8 @@ import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; import { materializePlugin } from '../plugin-store.js'; -import { satisfiesRange } from '../semver-range.js'; -/** The versions a plugin's `compat` is checked against: this SDK and the surface it pins. */ -export function runtimeVersions(): { sdk: string; surface: string } { - const pkg = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')) as { version: string; dependencies: Record }; - return { sdk: pkg.version, surface: pkg.dependencies['@relayflows/surface']! }; -} - -export function assertCompatible(manifest: FlowExtensionManifest, versions: { sdk: string; surface: string }): void { - for (const [what, range, actual] of [['surface', manifest.compat.surface, versions.surface], ['sdk', manifest.compat.sdk, versions.sdk]] as const) { - if (!satisfiesRange(actual, range)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${what} ${range}; this runtime has ${actual}.`); - } -} +export { assertCompatible, runtimeVersions }; /** Parse and validate the manifest inside a fetched plugin, checking that any self-declared source is the one it came from. */ export function extensionManifestOf(plugin: FetchedPlugin): { manifest: FlowExtensionManifest; manifestSha256: string } { @@ -55,7 +45,7 @@ export interface AddExtensionOptions { cwd?: string; fetch?: FetchLike; now?: () => Date; - versions?: { sdk: string; surface: string }; + versions?: RuntimeVersions; } /** diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index bb64a1b37..55c27a66d 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -5,6 +5,7 @@ import { preflightProviderTriggers } from '../provider-trigger-contract.js'; import { scheduleLowering } from '../schedule-trigger.js'; import { checkSlackHelpers } from '../slack-preflight.js'; import { flowRequirements } from '../flow-requirements.js'; +import { PluginError } from '../plugin-manifest.js'; import { inputFailureReport, readProjectConfig, type CheckReport } from './check.js'; /** @@ -40,6 +41,11 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const lowering = scheduleLowering(definition.name, trigger); return [{ handler, ...lowering }]; }); + // `?? []` tolerates the partial loader doubles the direct-run tests install. + const extensions = (loaded.extensions ?? []).map(extension => ({ + name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, + handlers: extension.handlers.length, + })); return { loaded, report: { @@ -47,11 +53,16 @@ export async function checkAuthoredTriggers(path: string): Promise<{ ok: !diagnostics.some(diagnostic => diagnostic.severity === 'refusal'), path, gates: [], resolutions: [], diagnostics, ...(schedules.length === 0 ? {} : { schedules }), + ...(extensions.length === 0 ? {} : { extensions }), requirements: flowRequirements(definition, { projectCli: config.cli }), ...(config.path === undefined ? {} : { projectConfigPath: config.path }), }, }; } catch (error) { + // A flow-extension refusal keeps its own code (plugin_source_drift, + // plugin_incompatible, …): the operator needs to know which record + // disagreed, not that "the spec is invalid". + if (error instanceof PluginError) return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; return { report: inputFailureReport({ kind: typeof error === 'object' && error !== null && 'kind' in error && error.kind === 'config_invalid' ? 'config_invalid' : 'invalid_spec', diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 22c362c9c..8a6c195a3 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -61,9 +61,21 @@ export interface CheckReport { schedules?: ScheduleInspection[]; /** Integrations, harnesses and MCP servers the flow declares it needs (`flow-requirements.ts`). */ requirements?: FlowRequirements; + /** Schema-2 flow extensions composed onto the authored flow, in lock order (`flow-extension-loader.ts`). */ + extensions?: ExtensionInspection[]; diagnostics: Array; } +export interface ExtensionInspection { + name: string; + version: string; + /** Canonical `github:/@#`. */ + ref: string; + digest: string; + /** How many `.on()` handlers it appends after the base flow's own. */ + handlers: number; +} + export interface ScheduleInspection { /** Position among the flow's handlers, so two identical declarations stay distinct. */ handler: number; diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts index 768c986c2..045d4841a 100644 --- a/packages/sdk/src/cli/plugin.ts +++ b/packages/sdk/src/cli/plugin.ts @@ -1,7 +1,6 @@ -import { readFileSync } from 'node:fs'; import type { CliIo } from '../cli.js'; import { fetchGithubPlugin, type FetchLike } from '../plugin-github.js'; -import { lockedPlugins, readPluginLock, type PluginLockEntry } from '../plugin-lock.js'; +import { lockedPlugins, readPluginLock, reconcileDeclaredExtensions, type PluginLockEntry } from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; import { pluginStoreDirectory, verifyStoredPlugin } from '../plugin-store.js'; @@ -24,30 +23,14 @@ export function parsePluginArgs(args: readonly string[]): PluginArgs | undefined return sub === 'list' ? { command: 'plugin', sub, json } : { command: 'plugin', sub, json, offline }; } -function declaredRefs(root: string): readonly string[] { - let config: { plugins?: unknown }; - try { config = JSON.parse(readFileSync(`${root}/flows.json`, 'utf8')); } - catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } - if (config === null || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every(p => typeof p === 'string')))) { - throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be strings.'); - } - return (config.plugins as string[] | undefined) ?? []; -} - /** * Cross-check the three records that must agree: `flows.json.plugins` * (declaration), `flows.lock.json` (provenance), and `.flows/plugins` (bytes). * With the network, the pinned commit is re-fetched and re-hashed too. */ export async function verifyPlugins(root: string, options: { offline: boolean; fetch?: FetchLike }): Promise<{ entry: PluginLockEntry; ref: string; directory: string; remote: 'verified' | 'skipped' }[]> { - const declared = declaredRefs(root).filter(ref => ref.startsWith('github:')); - const locked = lockedPlugins(readPluginLock(root)); - const lockedRefs = locked.map(p => p.ref); - for (const ref of declared) if (!lockedRefs.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.json declares ${ref} but flows.lock.json has no entry for it.`); - for (const ref of lockedRefs) if (!declared.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.lock.json records ${ref} but flows.json does not declare it.`); - if (declared.some((ref, index) => lockedRefs[index] !== ref)) throw new PluginError('plugin_lock_invalid', 'flows.lock.json order differs from flows.json.plugins.'); const results = []; - for (const { ref, entry, source } of locked) { + for (const { ref, entry, source } of reconcileDeclaredExtensions(root)) { const directory = pluginStoreDirectory(root, entry.name, entry.digest); await verifyStoredPlugin(directory, entry.digest); let remote: 'verified' | 'skipped' = 'skipped'; diff --git a/packages/sdk/src/cloud-deploy.ts b/packages/sdk/src/cloud-deploy.ts index 01873427b..6cf17520a 100644 --- a/packages/sdk/src/cloud-deploy.ts +++ b/packages/sdk/src/cloud-deploy.ts @@ -170,6 +170,12 @@ export async function deployToCloud( throw new CloudFlowError('unsupported_source', `${input.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}`); } + if (loaded.extensions.length > 0) { + // The deploy body carries one source file; a composed handler set has no + // wire form yet, so a deployment would silently lose the extensions. + throw new CloudFlowError('unsupported_source', + `Cloud deploy does not yet accept flow extensions (${loaded.extensions.map(e => e.name).join(', ')} composed by flows.json); deploy the base flow from a project without them.`); + } if (loaded.graph.length !== 1) { throw new CloudFlowError('unsupported_source', 'Cloud deploys one self-contained .flow.ts source without use dependencies.'); diff --git a/packages/sdk/src/cloud-run.ts b/packages/sdk/src/cloud-run.ts index 2223c9bf7..cf60dc32c 100644 --- a/packages/sdk/src/cloud-run.ts +++ b/packages/sdk/src/cloud-run.ts @@ -122,6 +122,10 @@ export async function prepareCloudSubmission( `${flow.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}. ` + 'Run `flows check` on it from the same directory.'); } + if (loaded.extensions.length > 0) { + throw new CloudFlowError('unsupported_source', + `Cloud authored submission does not yet accept flow extensions (${loaded.extensions.map(e => e.name).join(', ')} composed by flows.json).`); + } if (loaded.graph.length !== 1) { throw new CloudFlowError('unsupported_source', 'Cloud authored submission currently accepts one self-contained .flow.ts source without use dependencies.'); diff --git a/packages/sdk/src/flow-extension-compat.ts b/packages/sdk/src/flow-extension-compat.ts new file mode 100644 index 000000000..53af83400 --- /dev/null +++ b/packages/sdk/src/flow-extension-compat.ts @@ -0,0 +1,38 @@ +import { readFileSync } from 'node:fs'; +import type { FlowExtensionManifest } from './flow-extension-manifest.js'; +import { PluginError } from './plugin-manifest.js'; +import { satisfiesRange } from './semver-range.js'; + +export interface RuntimeVersions { readonly sdk: string; readonly surface: string } + +/** The versions a plugin's `compat` is checked against: this SDK and the surface it pins. */ +export function runtimeVersions(): RuntimeVersions { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version: string; dependencies: Record }; + return { sdk: pkg.version, surface: pkg.dependencies['@relayflows/surface']! }; +} + +/** `compat.surface` / `compat.sdk` against the runtime: a miss is a refusal, never a warning. */ +export function assertCompatible(manifest: FlowExtensionManifest, versions: RuntimeVersions): void { + for (const [what, range, actual] of [['surface', manifest.compat.surface, versions.surface], ['sdk', manifest.compat.sdk, versions.sdk]] as const) { + if (!satisfiesRange(actual, range)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${what} ${range}; this runtime has ${actual}.`); + } +} + +/** + * `compat.base` against the flow being extended. The surface's `FlowHeader` + * carries no version field, so a base flow has no version to satisfy a range + * with: only `*` can be met today, and anything narrower is refused rather + * than assumed. When the header grows a `version`, this is the one place to + * read it. + */ +export function assertBaseCompatible(manifest: FlowExtensionManifest, base: { readonly name: string; readonly version?: string }): void { + const entry = manifest.compat.base.find(b => b.name === base.name); + if (entry === undefined) { + throw new PluginError('plugin_incompatible', `${manifest.name} extends ${manifest.compat.base.map(b => b.name).join(', ')}, not "${base.name}".`); + } + if (base.version === undefined) { + if (entry.version !== '*') throw new PluginError('plugin_incompatible', `${manifest.name} requires ${base.name} ${entry.version}, but the base flow declares no version; only "*" can be satisfied.`); + return; + } + if (!satisfiesRange(base.version, entry.version)) throw new PluginError('plugin_incompatible', `${manifest.name} requires ${base.name} ${entry.version}; the base flow is ${base.version}.`); +} diff --git a/packages/sdk/src/flow-extension-loader.ts b/packages/sdk/src/flow-extension-loader.ts new file mode 100644 index 000000000..45301f2e5 --- /dev/null +++ b/packages/sdk/src/flow-extension-loader.ts @@ -0,0 +1,153 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { AuthoredFlowDefinition, FlowHandle } from './authored-flow.js'; +import { sha256 } from './bundle.js'; +import { assertBaseCompatible, assertCompatible, runtimeVersions, type RuntimeVersions } from './flow-extension-compat.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from './flow-extension-manifest.js'; +import { findPluginProject } from './plugin-loader.js'; +import { reconcileDeclaredExtensions, type PluginLockEntry } from './plugin-lock.js'; +import { PluginError } from './plugin-manifest.js'; +import { pluginStoreDirectory, verifyStoredPlugin } from './plugin-store.js'; + +/** + * Compose schema-2 flow extensions onto a base authored flow. + * + * Order of operations is the security argument, so it is fixed: + * 1. `flows.json.plugins` and `flows.lock.json` must agree (declaration ⇔ + * provenance, same order); + * 2. the materialized store is re-hashed against the lock's digest and the + * manifest bytes against the lock's manifest hash — nothing under + * `.flows/plugins` is read as code before this passes; + * 3. the manifest is validated, its compat checked against the runtime and + * the base flow, and anything this slice does not compose (hooks, `use`, + * schedule triggers, non-provider webhooks) is refused; + * 4. only then is the entry imported, and its handlers are checked against + * the manifest's declared triggers — an entry cannot subscribe to more + * than it declared. + * Extensions compose after the base, in lock order; nothing replaces, + * reorders, or widens a base handler. + */ +type TriggerHandler = AuthoredFlowDefinition['handlers'][number]; + +export interface LoadedFlowExtension { + readonly name: string; + readonly version: string; + readonly ref: string; + readonly digest: string; + readonly directory: string; + readonly entryPath: string; + readonly manifest: FlowExtensionManifest; + readonly handle: FlowHandle; + readonly handlers: readonly TriggerHandler[]; +} + +export interface ImportedFlow { + readonly handle: FlowHandle; + readonly getDefinition: (handle: FlowHandle) => AuthoredFlowDefinition; + readonly surfaceAuthority: Authority; +} + +export interface LoadFlowExtensionsOptions { + readonly importFlow: (path: string) => Promise>; + readonly sameAuthority: (a: Authority, b: Authority) => boolean; + readonly versions?: RuntimeVersions; +} + +const EXTENSION_HEADER_FIELDS = new Set(['budget', 'tools']); + +function unsupported(name: string, what: string): never { + throw new PluginError('plugin_unsupported', `${name}: ${what} is not composed by this release.`); +} + +/** The `{provider, event, action?}` a provider subscription lowers to, or undefined for anything else. */ +function subscriptionOf(handler: TriggerHandler): { provider: string; event: string; action?: string } | undefined { + const trigger = handler.trigger; + if (trigger.kind !== 'webhook' || trigger.filter === undefined) return undefined; + const { provider, type, payload } = trigger.filter as { provider?: unknown; type?: unknown; payload?: unknown }; + if (typeof provider !== 'string' || provider !== trigger.name || typeof type !== 'string') return undefined; + const action = typeof payload === 'object' && payload !== null && !Array.isArray(payload) ? (payload as { action?: unknown }).action : undefined; + if (action !== undefined && typeof action !== 'string') return undefined; + return action === undefined ? { provider, event: type } : { provider, event: type, action }; +} + +function assertDeclaredSubscription(name: string, manifest: FlowExtensionManifest, handler: TriggerHandler, index: number): void { + const subscription = subscriptionOf(handler); + if (handler.trigger.kind === 'schedule') unsupported(name, `handler ${index} (a schedule trigger)`); + if (subscription === undefined) { + throw new PluginError('plugin_manifest_invalid', `${name}: handler ${index} is not a provider subscription; extension handlers must be provider triggers declared in flows-plugin.json.`); + } + const declared = manifest.triggers.find(t => t.provider === subscription.provider && t.event === subscription.event); + const covered = declared !== undefined && (subscription.action === undefined ? declared.actions.length === 0 : declared.actions.includes(subscription.action)); + if (!covered) { + const spelled = `${subscription.provider} ${subscription.event}${subscription.action === undefined ? '' : `.${subscription.action}`}`; + throw new PluginError('plugin_manifest_invalid', `${name}: handler ${index} subscribes to ${spelled}, which flows-plugin.json does not declare in triggers.`); + } +} + +async function loadOne( + root: string, lock: PluginLockEntry, ref: string, + base: { readonly definition: AuthoredFlowDefinition; readonly surfaceAuthority: Authority }, + options: LoadFlowExtensionsOptions, +): Promise { + const directory = pluginStoreDirectory(root, lock.name, lock.digest); + await verifyStoredPlugin(directory, lock.digest); + const manifestBytes = readFileSync(join(directory, 'flows-plugin.json')); + if (sha256(manifestBytes) !== lock.manifestSha256) throw new PluginError('plugin_source_drift', `${ref}: flows-plugin.json differs from the lockfile's manifest hash.`); + let input: unknown; + try { input = JSON.parse(manifestBytes.toString('utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', `${ref}: flows-plugin.json is not valid JSON.`); } + const manifest = validateFlowExtensionManifest(input); + if (manifest.name !== lock.name || manifest.version !== lock.version) throw new PluginError('plugin_source_drift', `${ref}: manifest names ${manifest.name}@${manifest.version}, lockfile has ${lock.name}@${lock.version}.`); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + assertBaseCompatible(manifest, { name: base.definition.name }); + if (manifest.extends.hooks.length > 0) unsupported(manifest.name, `hooks (${manifest.extends.hooks.join(', ')})`); + const baseBudget = base.definition.header.budget; + const ceiling = manifest.permissions.budget; + if (ceiling?.dollars !== undefined && typeof baseBudget === 'object' && baseBudget.dollars !== undefined && ceiling.dollars > baseBudget.dollars) { + throw new PluginError('plugin_incompatible', `${manifest.name} declares a $${ceiling.dollars} budget ceiling above the base flow's $${baseBudget.dollars}.`); + } + const entryPath = join(directory, manifest.entry); + let imported: ImportedFlow; + try { imported = await options.importFlow(entryPath); } + catch (error) { throw new PluginError('plugin_manifest_invalid', `${ref}: entry ${manifest.entry} did not load: ${error instanceof Error ? error.message : String(error)}`); } + if (!options.sameAuthority(imported.surfaceAuthority, base.surfaceAuthority)) { + throw new PluginError('plugin_incompatible', `${manifest.name}: entry resolves a different @relayflows/surface than the base flow.`); + } + const definition = imported.getDefinition(imported.handle); + const foreign = Object.keys(definition.header).filter(key => !EXTENSION_HEADER_FIELDS.has(key)); + if (foreign.length > 0) unsupported(manifest.name, `entry header ${foreign.join(', ')}`); + if (manifest.extends.handlers && definition.handlers.length === 0) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: extends.handlers is true but ${manifest.entry} declares no .on() handlers.`); + } + if (!manifest.extends.handlers && definition.handlers.length > 0) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: ${manifest.entry} declares handlers but extends.handlers is false.`); + } + definition.handlers.forEach((handler, index) => assertDeclaredSubscription(manifest.name, manifest, handler, index)); + return Object.freeze({ + name: manifest.name, version: manifest.version, ref, digest: lock.digest, directory, entryPath, manifest, + handle: imported.handle, handlers: Object.freeze([...definition.handlers]), + }); +} + +/** Extensions declared by the project that owns `flowPath`, verified and loaded in lock order; empty when none are declared. */ +export async function loadFlowExtensions( + flowPath: string, + base: { readonly definition: AuthoredFlowDefinition; readonly surfaceAuthority: Authority }, + options: LoadFlowExtensionsOptions, +): Promise { + const root = findPluginProject(dirname(flowPath)); + if (root === undefined) return Object.freeze([]); + const declared = reconcileDeclaredExtensions(root); + const loaded: LoadedFlowExtension[] = []; + for (const { ref, entry } of declared) loaded.push(await loadOne(root, entry, ref, base, options)); + return Object.freeze(loaded); +} + +/** The base definition with extension handlers appended in lock order; the base's own fields are untouched. */ +export function composeDefinition(base: AuthoredFlowDefinition, extensions: readonly LoadedFlowExtension[]): AuthoredFlowDefinition { + if (extensions.length === 0) return base; + return Object.freeze({ + ...base, + handlers: Object.freeze([...base.handlers, ...extensions.flatMap(extension => extension.handlers)]), + }); +} diff --git a/packages/sdk/src/plugin-loader.ts b/packages/sdk/src/plugin-loader.ts index e4352b0b5..797a89e6a 100644 --- a/packages/sdk/src/plugin-loader.ts +++ b/packages/sdk/src/plugin-loader.ts @@ -48,13 +48,11 @@ export async function loadPlugins(start: string): Promise typeof p === 'string')))) { throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be package names.'); } - // Flow extensions (github:… entries) are declared and locked by `flows add` - // but not yet composed at run time. Refuse before any helper loads rather - // than silently running the base flow without them. - const extension = (config.plugins as string[] | undefined)?.find(isGithubPluginRef); - if (extension !== undefined) throw new PluginError('plugin_unsupported', `${extension} is a flow-extension plugin; runtime composition of flow extensions is not supported by this release.`); + // Flow extensions (github:… entries) are not helpers: the authored flow + // loader verifies and composes them (flow-extension-loader.ts). Here they + // are simply not helper packages, so they are left out of the helper set. const scope = join(root, 'node_modules/@flows'); - const names = new Set((config.plugins as string[] | undefined)?.map(pluginPackageName)); + const names = new Set((config.plugins as string[] | undefined)?.filter(p => !isGithubPluginRef(p)).map(pluginPackageName)); if (existsSync(scope)) for (const name of readdirSync(scope).sort()) { if (name.startsWith('helper-') && !names.has(`@flows/${name}`)) { throw new PluginError('plugin_unlisted', `@flows/${name} is installed but not declared in flows.json plugins. Run flows add ${name}.`); diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index ee6f00184..94bbc208d 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { PluginError } from './plugin-manifest.js'; -import { SHA, canonicalPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; +import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; /** * `flows.lock.json` — the project's plugin provenance. `flows.json.plugins` @@ -23,7 +23,11 @@ export interface PluginLockEntry { readonly digest: string; /** sha256 of the `flows-plugin.json` bytes as installed. */ readonly manifestSha256: string; - /** 1-based position in `flows.json.plugins`; the deterministic composition order. */ + /** + * 1-based position among the flow-extension (`github:`) entries of + * `flows.json.plugins`, in declaration order. Helper entries interspersed in + * that list do not count, so the order is the composition order exactly. + */ readonly order: number; readonly resolvedAt: string; } @@ -97,3 +101,31 @@ export function lockedPlugins(lock: PluginLock): readonly { ref: string; entry: return { ref: canonicalPluginRef(source), entry, source }; }); } + +/** The `github:` entries of `flows.json.plugins`, in declaration order; helper entries are left out. */ +export function declaredExtensionRefs(root: string): readonly string[] { + let config: { plugins?: unknown }; + try { config = JSON.parse(readFileSync(join(root, 'flows.json'), 'utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } + if (config === null || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every(p => typeof p === 'string')))) { + throw new PluginError('plugin_manifest_invalid', 'flows.json plugins must be strings.'); + } + return Object.freeze(((config.plugins as string[] | undefined) ?? []).filter(isGithubPluginRef)); +} + +/** + * The three records that must agree before an extension is trusted: + * `flows.json.plugins` (declaration), `flows.lock.json` (provenance), and — + * checked by the caller against the returned digests — `.flows/plugins` + * (bytes). Any declaration without a lock entry, lock entry without a + * declaration, or order disagreement is `plugin_lock_invalid`. + */ +export function reconcileDeclaredExtensions(root: string): readonly { ref: string; entry: PluginLockEntry; source: PluginSourceRef }[] { + const declared = declaredExtensionRefs(root); + const locked = lockedPlugins(readPluginLock(root)); + const lockedRefs = locked.map(p => p.ref); + for (const ref of declared) if (!lockedRefs.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.json declares ${ref} but flows.lock.json has no entry for it.`); + for (const ref of lockedRefs) if (!declared.includes(ref)) throw new PluginError('plugin_lock_invalid', `flows.lock.json records ${ref} but flows.json does not declare it.`); + if (declared.some((ref, index) => lockedRefs[index] !== ref)) throw new PluginError('plugin_lock_invalid', 'flows.lock.json order differs from flows.json.plugins.'); + return locked; +} diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts new file mode 100644 index 000000000..30d262445 --- /dev/null +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -0,0 +1,190 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; +import { checkAuthoredTriggers } from '../src/cli/check-triggers.js'; +import { runCli } from '../src/cli.js'; +import { readPluginLock } from '../src/plugin-lock.js'; +import { loadPlugins } from '../src/plugin-loader.js'; +import { pluginStoreDirectory } from '../src/plugin-store.js'; +import { preflightProviderTriggers } from '../src/provider-trigger-contract.js'; +import { SHA_A, SHA_B, entriesFromDirectory, fakeGithub, type FakeEntry } from './fake-github.js'; + +const fixtureRoot = resolve('../../testdata/plugins'); +const babysitter = entriesFromDirectory(join(fixtureRoot, 'extension-babysitter'), 'examples/babysitter'); +const manifestJson = JSON.parse(readFileSync(join(fixtureRoot, 'extension-babysitter/flows-plugin.json'), 'utf8')); +const REF = `github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`; +const versions = { sdk: '2.0.22', surface: '2.0.22' }; +const now = () => new Date('2026-09-20T12:00:00Z'); +const dirs: string[] = []; +afterEach(() => { dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); }); + +const BASE = ` + import { flow, github } from '@relayflows/surface'; + export default flow('software-factory', { budget: { dollars: 10, wallclock: '1h' } }, async f => { f.done('success'); }) + .on(github.issues({ action: 'opened' }), async f => { f.done('success'); }); +`; + +/** A project the way an operator has one: flows.json, the base flow, and a resolvable surface. */ +function project(base = BASE) { + const cwd = mkdtempSync(join(tmpdir(), 'flow-compose-')); dirs.push(cwd); + mkdirSync(join(cwd, 'node_modules/@relayflows'), { recursive: true }); + symlinkSync(resolve('node_modules/@relayflows/surface'), join(cwd, 'node_modules/@relayflows/surface')); + writeFileSync(join(cwd, 'package.json'), '{"type":"module"}'); + writeFileSync(join(cwd, 'flows.json'), JSON.stringify({ cli: 'claude', executors: ['github'] })); + writeFileSync(join(cwd, 'software-factory.flow.ts'), base); + const messages: string[] = []; + const io = { stdout: (s: string) => messages.push(s), stderr: (s: string) => messages.push(s) }; + return { cwd, io, messages, text: () => messages.join('\n'), flow: join(cwd, 'software-factory.flow.ts') }; +} +function repo(entries: FakeEntry[]) { + return fakeGithub({ 'AgentWorkforce/flows': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries } } } }).fetch; +} +function variant(patch: (m: Record) => unknown, entry?: string): FakeEntry[] { + return babysitter.map(e => { + if (e.path.endsWith('flows-plugin.json')) return { ...e, data: Buffer.from(JSON.stringify(patch(structuredClone(manifestJson)))) }; + if (entry !== undefined && e.path.endsWith('babysitter.flow.ts')) return { ...e, data: Buffer.from(entry) }; + return e; + }); +} +async function install(p: ReturnType, entries: FakeEntry[] = babysitter, ref = REF, fetch = repo(entries)) { + expect(await addExtensionPlugin(ref, p.io, { cwd: p.cwd, fetch, now, versions })).toBe(0); + p.messages.length = 0; +} +const subscriptions = (loaded: Awaited>) => + loaded.getDefinition(loaded.handle).handlers.map(h => { + const f = h.trigger.kind === 'webhook' ? h.trigger.filter as { type: string; payload?: { action?: string } } : undefined; + return f === undefined ? h.trigger.kind : `${f.type}${f.payload?.action === undefined ? '' : `.${f.payload.action}`}`; + }); + +describe('composing flow extensions onto a base flow', () => { + it('appends the Babysitter handler surface after the base, in lock order, without touching the base definition', async () => { + const p = project(); + await install(p); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + expect(loaded.extensions.map(e => ({ name: e.name, ref: e.ref, handlers: e.handlers.length }))).toEqual([{ name: 'babysitter', ref: REF, handlers: 8 }]); + expect(subscriptions(loaded)).toEqual([ + 'issues.opened', + 'pull_request.opened', 'pull_request.synchronize', 'pull_request.reopened', 'pull_request.closed', + 'pull_request_review.submitted', 'pull_request_review.dismissed', 'check_run.completed', 'issue_comment.created', + ]); + const composed = loaded.getDefinition(loaded.handle); + expect(Object.isFrozen(composed) && Object.isFrozen(composed.handlers)).toBe(true); + // The base's own definition, as its surface copy holds it, is unchanged. + expect(loaded.graph[0]!.getDefinition(loaded.handle).handlers).toHaveLength(1); + expect(composed.name).toBe('software-factory'); + expect(composed.header).toBe(loaded.graph[0]!.getDefinition(loaded.handle).header); + expect(loaded.graph.map(node => node.handle.name)).toEqual(['software-factory', 'babysitter']); + expect(loaded.graph[1]!.path).toBe(join(pluginStoreDirectory(p.cwd, 'babysitter', loaded.extensions[0]!.digest), 'babysitter.flow.ts')); + // Every composed subscription is one the surface registry can lower. + expect(preflightProviderTriggers(composed.handlers.map(h => h.trigger))).toEqual([]); + // The extension's own handle is not the root: asking for its definition goes to the surface, not the composition. + expect(loaded.getDefinition(loaded.extensions[0]!.handle).handlers).toHaveLength(8); + }); + it('loads the root alone with extensions: none, and helper loading ignores extension entries', async () => { + const p = project(); + await install(p); + const alone = await loadAuthoredFlow(p.flow, { extensions: 'none' }); + expect(alone.extensions).toEqual([]); + expect(subscriptions(alone)).toEqual(['issues.opened']); + expect(await loadPlugins(p.cwd)).toEqual([]); + }); + it('composes two extensions in declaration order, and the order is the lockfile order', async () => { + const second = variant(m => ({ ...m, name: 'second', triggers: [{ provider: 'github', event: 'issues', actions: ['closed'] }] }), + "import { flow, github } from '@relayflows/surface';\nexport default flow('second', async f => { f.done('success'); }).on(github.issues({ action: 'closed' }), async f => { f.done('success'); });\n") + .map(e => ({ ...e, path: e.path.replace('examples/babysitter', 'examples/second') })); + const fetch = fakeGithub({ 'AgentWorkforce/flows': { refs: { main: SHA_A }, commits: { [SHA_A]: { entries: babysitter }, [SHA_B]: { entries: second } } } }).fetch; + const SECOND = `github:AgentWorkforce/flows@${SHA_B}#examples/second`; + const forward = project(); + await install(forward, babysitter, REF, fetch); + await install(forward, second, SECOND, fetch); + expect(readPluginLock(forward.cwd).plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter'], [2, 'second']]); + const loadedForward = await loadAuthoredFlow(forward.flow, { versions }); + expect(loadedForward.extensions.map(e => e.name)).toEqual(['babysitter', 'second']); + expect(subscriptions(loadedForward).at(-1)).toBe('issues.closed'); + const reverse = project(); + await install(reverse, second, SECOND, fetch); + await install(reverse, babysitter, REF, fetch); + const loadedReverse = await loadAuthoredFlow(reverse.flow, { versions }); + expect(loadedReverse.extensions.map(e => e.name)).toEqual(['second', 'babysitter']); + expect(subscriptions(loadedReverse).slice(0, 2)).toEqual(['issues.opened', 'issues.closed']); + }); + it('flows check reports the composition and keeps the composed triggers deliverable', async () => { + const p = project(); + await install(p); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(true); + expect(report.extensions).toEqual([{ name: 'babysitter', version: '0.1.0', ref: REF, digest: expect.stringMatching(/^[0-9a-f]{64}$/), handlers: 8 }]); + expect(report.requirements?.integrations.map(i => i.provider)).toContain('github'); + expect(await runCli(['check', p.flow], p.io)).toBe(0); + expect(p.text()).toContain(`EXTENSION babysitter@0.1.0 ${REF} sha256:`); + expect(p.text()).toContain('8 handler(s) composed after the base flow'); + }); +}); + +describe('composition fails closed', () => { + it('refuses a tampered store before importing the entry', async () => { + const p = project(); + await install(p); + const store = pluginStoreDirectory(p.cwd, 'babysitter', readPluginLock(p.cwd).plugins[0]!.digest); + writeFileSync(join(store, 'babysitter.flow.ts'), "throw new Error('the entry must not be imported before the digest check');"); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_source_drift' }); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(false); + expect(report.diagnostics[0]).toMatchObject({ kind: 'plugin_source_drift' }); + }); + it('refuses when flows.json and the lockfile disagree', async () => { + const p = project(); + await install(p); + writeFileSync(join(p.cwd, 'flows.lock.json'), JSON.stringify({ version: 2, plugins: [] })); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_lock_invalid' }); + }); + it('refuses a runtime the extension declares itself incompatible with', async () => { + const p = project(); + await install(p); + await expect(loadAuthoredFlow(p.flow, { versions: { sdk: '2.0.22', surface: '3.0.0' } })).rejects.toMatchObject({ code: 'plugin_incompatible', message: expect.stringContaining('requires surface ^2.0.22') }); + }); + it.each([ + ['a base flow it does not extend', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'other-flow', version: '*' }] } }), 'plugin_incompatible', 'not "software-factory"'], + ['a base version range the unversioned base cannot satisfy', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^1.0.0' }] } }), 'plugin_incompatible', 'declares no version'], + ['a budget ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 11 } } }), 'plugin_incompatible', 'above the base flow'], + ['hooks, which this release does not compose', (m: Record) => ({ ...m, extends: { handlers: true, hooks: ['merge-gate'] } }), 'plugin_unsupported', 'hooks (merge-gate)'], + ])('refuses %s', async (_, patch, code, message) => { + const p = project(); + await install(p, variant(patch)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code, message: expect.stringContaining(message) }); + }); + it.each([ + ['an entry subscribing beyond its manifest', undefined, + "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); }).on(github.pull_request('labeled'), async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'subscribes to github pull_request.labeled'], + ['an entry with a schedule handler', undefined, + "import { flow, schedule } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(schedule.every('1h'), async f => { f.done('success'); });\n", + 'plugin_unsupported', 'schedule trigger'], + ['an entry with a generic webhook handler', undefined, + "import { flow, webhook } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(webhook('deploys', { provider: 'aws' }), async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'not a provider subscription'], + ['an entry that declares no handlers although the manifest says it does', undefined, + "import { flow } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); });\n", + 'plugin_manifest_invalid', 'declares no .on() handlers'], + ['an entry that uses the use: header', undefined, + "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', { use: ['./other.flow.ts'] }, async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); });\n", + 'plugin_unsupported', 'entry header use'], + ['an entry that does not default-export flow()', undefined, + "export default { name: 'forged' };\n", + 'plugin_manifest_invalid', 'did not load'], + ])('refuses %s', async (_, __, entry, code, message) => { + const p = project(); + await install(p, variant(m => m, entry)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code, message: expect.stringContaining(message) }); + }); + it('never composes an event the surface registry cannot lower, even if an entry asks for it', async () => { + // The manifest gate refuses ready_for_review at install; an entry alone cannot smuggle it past the manifest. + const p = project(); + const entries = variant(m => m, "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('ready_for_review'), async f => { f.done('success'); });\n"); + await install(p, entries); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code: 'plugin_manifest_invalid', message: expect.stringContaining('pull_request.ready_for_review') }); + }); +}); diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts index ca252b64c..75e2f6daf 100644 --- a/packages/sdk/tests/plugin-extension.test.ts +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -175,7 +175,7 @@ describe('bounded, verified fetches', () => { describe('schema-2 manifest validation', () => { it('accepts the worked Babysitter manifest and freezes it', () => { const m = validateFlowExtensionManifest(manifestJson); - expect(m).toMatchObject({ schema: 2, kind: 'flow-extension', name: 'babysitter', entry: 'babysitter.flow.ts', extends: { handlers: true, hooks: ['merge-gate'] } }); + expect(m).toMatchObject({ schema: 2, kind: 'flow-extension', name: 'babysitter', entry: 'babysitter.flow.ts', extends: { handlers: true, hooks: [] } }); expect(m.triggers).toHaveLength(4); expect(Object.isFrozen(m) && Object.isFrozen(m.permissions) && Object.isFrozen(m.triggers)).toBe(true); }); @@ -274,11 +274,17 @@ describe('legacy helper plugins are untouched', () => { expect(gh.calls).toEqual([]); expect(p.text()).toContain('plugin_install_failed'); }); - it('refuses runtime composition of a declared flow extension, fail closed, before loading helpers', async () => { + it('keeps the helper loader helper-only: extension entries are neither helpers nor unlisted packages', async () => { const gh = github(); const p = project(); await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); - await expect(loadPlugins(p.cwd)).rejects.toMatchObject({ code: 'plugin_unsupported', message: expect.stringContaining(REF) }); - expect(await loadPlugins(project().cwd)).toEqual([]); + // Extensions are composed by the authored flow loader (flow-extension-compose.test.ts); here they are simply not helpers. + expect(await loadPlugins(p.cwd)).toEqual([]); + vi.stubEnv('DATADOG_API_KEY', 'test'); + cpSync(join(fixtureRoot, 'helper-datadog'), join(p.cwd, 'node_modules/@flows/helper-datadog'), { recursive: true }); + const config = JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')); + writeFileSync(join(p.cwd, 'flows.json'), JSON.stringify({ ...config, plugins: ['helper-datadog', ...config.plugins] })); + expect((await loadPlugins(p.cwd)).map(plugin => plugin.manifest.name)).toEqual(['helper-datadog']); + expect(readPluginLock(p.cwd).plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter']]); }); it('dispatches through the CLI: add refuses a bad reference offline, plugin list and verify run', async () => { const p = project(); diff --git a/testdata/plugins/extension-babysitter/README.md b/testdata/plugins/extension-babysitter/README.md index 85c11cbef..101773d74 100644 --- a/testdata/plugins/extension-babysitter/README.md +++ b/testdata/plugins/extension-babysitter/README.md @@ -2,9 +2,14 @@ The worked schema-2 `kind: "flow-extension"` manifest for Babysitter on Software Garden, served to the SDK tests by a fake GitHub (see -`packages/sdk/tests/plugin-extension.test.ts`). It is a fixture, not an -installable example: the entry is a stub, and runtime composition of flow -extensions is refused with `plugin_unsupported` in this release. +`packages/sdk/tests/plugin-extension.test.ts` and +`tests/flow-extension-compose.test.ts`). It is a fixture, not an installable +example: the entry carries Babysitter's handler surface — one `.on()` per +declared subscription — over a body that only declines, so composition onto a +base flow can be proven without the real review body. `extends.hooks` is empty +because hooks are not composed by this release (a manifest declaring one is +refused with `plugin_unsupported`); the `merge-gate` hook from the design is a +later slice. Babysitter's own subscription contract (branch `feat/babysitter-v2`) names eleven GitHub subscriptions. Three of them — `pull_request.ready_for_review`, diff --git a/testdata/plugins/extension-babysitter/babysitter.flow.ts b/testdata/plugins/extension-babysitter/babysitter.flow.ts index 6b7d4bb2b..e27553f09 100644 --- a/testdata/plugins/extension-babysitter/babysitter.flow.ts +++ b/testdata/plugins/extension-babysitter/babysitter.flow.ts @@ -1,5 +1,17 @@ -// Offline fixture entry. The real Babysitter body lives on the babysitter -// branch of AgentWorkforce/flows; this stub exists so the manifest's `entry` -// resolves inside the fixture. Runtime composition is not implemented yet. -import { flow } from '@relayflows/surface'; -export default flow('babysitter', async (f) => { f.done('declined'); }); +// Offline fixture entry: the handler surface of Babysitter, one `.on()` per +// subscription the manifest declares, over a body that only declines. The real +// review body lives on the babysitter branch of AgentWorkforce/flows; this +// file exists so composition can be proven against the declared contract. +import { flow, github, type Ctx } from '@relayflows/surface'; + +async function babysit(f: Ctx): Promise { f.done('declined'); } + +export default flow('babysitter', { budget: { dollars: 8, wallclock: '45m' } }, babysit) + .on(github.pull_request('opened'), babysit) + .on(github.pull_request('synchronize'), babysit) + .on(github.pull_request('reopened'), babysit) + .on(github.pull_request('closed'), babysit) + .on(github.pull_request_review({ action: 'submitted' }), babysit) + .on(github.pull_request_review({ action: 'dismissed' }), babysit) + .on(github.check_run('completed'), babysit) + .on(github.issue_comment('created'), babysit); diff --git a/testdata/plugins/extension-babysitter/flows-plugin.json b/testdata/plugins/extension-babysitter/flows-plugin.json index c730c925d..5f7afe7eb 100644 --- a/testdata/plugins/extension-babysitter/flows-plugin.json +++ b/testdata/plugins/extension-babysitter/flows-plugin.json @@ -7,36 +7,119 @@ "compat": { "surface": "^2.0.22", "sdk": "^2.0.22", - "base": [{ "name": "software-factory", "version": "*" }] + "base": [ + { + "name": "software-factory", + "version": "*" + } + ] }, "entry": "babysitter.flow.ts", - "extends": { "handlers": true, "hooks": ["merge-gate"] }, + "extends": { + "handlers": true, + "hooks": [] + }, "triggers": [ - { "provider": "github", "event": "pull_request", "actions": ["opened", "synchronize", "reopened", "closed"] }, - { "provider": "github", "event": "pull_request_review", "actions": ["submitted", "dismissed"] }, - { "provider": "github", "event": "check_run", "actions": ["completed"] }, - { "provider": "github", "event": "issue_comment", "actions": ["created"] } + { + "provider": "github", + "event": "pull_request", + "actions": [ + "opened", + "synchronize", + "reopened", + "closed" + ] + }, + { + "provider": "github", + "event": "pull_request_review", + "actions": [ + "submitted", + "dismissed" + ] + }, + { + "provider": "github", + "event": "check_run", + "actions": [ + "completed" + ] + }, + { + "provider": "github", + "event": "issue_comment", + "actions": [ + "created" + ] + } ], "permissions": { - "integrations": ["github"], - "harnesses": ["claude"], + "integrations": [ + "github" + ], + "harnesses": [ + "claude" + ], "mcp": [], - "writes": ["github:pull_request:comment"], - "budget": { "dollars": 8, "wallclock": "45m" } + "writes": [ + "github:pull_request:comment" + ], + "budget": { + "dollars": 8, + "wallclock": "45m" + } + }, + "preflight": { + "credentials": [], + "servers": [ + "https://api.github.com" + ] }, - "preflight": { "credentials": [], "servers": ["https://api.github.com"] }, "config": { "type": "object", "properties": { - "testCommand": { "type": "string" }, - "botLogin": { "type": "string" }, - "approvers": { "type": "array", "items": { "type": "string" } }, - "organizations": { "type": "array", "items": { "type": "string" } }, - "merge": { "type": "boolean", "default": false }, - "skipLabels": { "type": "array", "items": { "type": "string" }, "default": ["no-agent-relay-review"] }, - "requiredChecks": { "type": "array", "items": { "type": "string" } } + "testCommand": { + "type": "string" + }, + "botLogin": { + "type": "string" + }, + "approvers": { + "type": "array", + "items": { + "type": "string" + } + }, + "organizations": { + "type": "array", + "items": { + "type": "string" + } + }, + "merge": { + "type": "boolean", + "default": false + }, + "skipLabels": { + "type": "array", + "items": { + "type": "string" + }, + "default": [ + "no-agent-relay-review" + ] + }, + "requiredChecks": { + "type": "array", + "items": { + "type": "string" + } + } }, - "required": ["testCommand", "approvers"], + "required": [ + "testCommand", + "approvers" + ], "additionalProperties": false } } From 841f9d535feac742c67eb7b621ba965c4f7aa315 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 15:29:23 -0700 Subject: [PATCH 03/17] test(sdk): compare canonical realpaths for the composed extension entry path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loader realpaths the root flow, and the extension store path derives from that root; on macOS the tmpdir is a symlink (/var → /private/var), so an absolute-path equality against the un-resolved tmpdir failed there while passing on Linux. Compare realpaths on both sides. Runtime checks unchanged. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 19498b5b-4a5c-4096-a978-84d7082bd5a4 --- packages/sdk/tests/flow-extension-compose.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 30d262445..42396492d 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; @@ -77,7 +77,9 @@ describe('composing flow extensions onto a base flow', () => { expect(composed.name).toBe('software-factory'); expect(composed.header).toBe(loaded.graph[0]!.getDefinition(loaded.handle).header); expect(loaded.graph.map(node => node.handle.name)).toEqual(['software-factory', 'babysitter']); - expect(loaded.graph[1]!.path).toBe(join(pluginStoreDirectory(p.cwd, 'babysitter', loaded.extensions[0]!.digest), 'babysitter.flow.ts')); + // Compare canonical paths: the loader realpaths the root (macOS tmpdir is a + // symlink, /var → /private/var), and the store path derives from that root. + expect(realpathSync(loaded.graph[1]!.path)).toBe(realpathSync(join(pluginStoreDirectory(p.cwd, 'babysitter', loaded.extensions[0]!.digest), 'babysitter.flow.ts'))); // Every composed subscription is one the surface registry can lower. expect(preflightProviderTriggers(composed.handlers.map(h => h.trigger))).toEqual([]); // The extension's own handle is not the root: asking for its definition goes to the surface, not the composition. From 45246a70e4bdbeb6672c01a8ff75b2e6b52cd7b3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 15:30:31 -0700 Subject: [PATCH 04/17] fix(sdk): extension graph nodes carry their own surface accessor authored-flow-loader appended extension nodes with the root's getDefinition, which answers only for the root's @relayflows/surface WeakMap; a node's accessor must be the one its own entry import returned. LoadedFlowExtension now records that accessor and the graph node uses it. The root's composed accessor is unchanged. Test: graph[1].getDefinition(graph[1].handle) resolves the babysitter definition with its eight handlers. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 19498b5b-4a5c-4096-a978-84d7082bd5a4 --- packages/sdk/src/authored-flow-loader.ts | 4 +++- packages/sdk/src/flow-extension-loader.ts | 4 +++- packages/sdk/tests/flow-extension-compose.test.ts | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/authored-flow-loader.ts b/packages/sdk/src/authored-flow-loader.ts index 22b6f8d28..3118815be 100644 --- a/packages/sdk/src/authored-flow-loader.ts +++ b/packages/sdk/src/authored-flow-loader.ts @@ -136,7 +136,9 @@ export async function loadAuthoredFlow(path: string, options: LoadAuthoredFlowOp const getDefinition: GetFlowDefinition = (handle: FlowHandle) => (handle === root.handle ? composed : root.getDefinition(handle)) as AuthoredFlowDefinition; for (const extension of extensions) { - graph.push(Object.freeze({ path: extension.entryPath, handle: extension.handle, getDefinition: root.getDefinition, + // The node carries the accessor the entry's own surface copy handed back: + // the root's accessor answers for the root's WeakMap only. + graph.push(Object.freeze({ path: extension.entryPath, handle: extension.handle, getDefinition: extension.getDefinition, surfaceAuthority: root.surfaceAuthority, use: Object.freeze([]) })); } return Object.freeze({ sourcePath: root.path, handle: root.handle, getDefinition, diff --git a/packages/sdk/src/flow-extension-loader.ts b/packages/sdk/src/flow-extension-loader.ts index 45301f2e5..5a7f6c3a3 100644 --- a/packages/sdk/src/flow-extension-loader.ts +++ b/packages/sdk/src/flow-extension-loader.ts @@ -38,6 +38,8 @@ export interface LoadedFlowExtension { readonly entryPath: string; readonly manifest: FlowExtensionManifest; readonly handle: FlowHandle; + /** Bound to the surface copy the entry itself imported; the base's accessor cannot see this handle's WeakMap entry. */ + readonly getDefinition: ImportedFlow['getDefinition']; readonly handlers: readonly TriggerHandler[]; } @@ -125,7 +127,7 @@ async function loadOne( definition.handlers.forEach((handler, index) => assertDeclaredSubscription(manifest.name, manifest, handler, index)); return Object.freeze({ name: manifest.name, version: manifest.version, ref, digest: lock.digest, directory, entryPath, manifest, - handle: imported.handle, handlers: Object.freeze([...definition.handlers]), + handle: imported.handle, getDefinition: imported.getDefinition, handlers: Object.freeze([...definition.handlers]), }); } diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 42396492d..8538cff82 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -84,6 +84,11 @@ describe('composing flow extensions onto a base flow', () => { expect(preflightProviderTriggers(composed.handlers.map(h => h.trigger))).toEqual([]); // The extension's own handle is not the root: asking for its definition goes to the surface, not the composition. expect(loaded.getDefinition(loaded.extensions[0]!.handle).handlers).toHaveLength(8); + // Its graph node resolves through the accessor its own entry import returned, not the root's. + const node = loaded.graph[1]!; + expect(node.getDefinition).toBe(loaded.extensions[0]!.getDefinition); + expect(node.getDefinition(node.handle).name).toBe('babysitter'); + expect(node.getDefinition(node.handle).handlers).toHaveLength(8); }); it('loads the root alone with extensions: none, and helper loading ignores extension entries', async () => { const p = project(); From 84e4c895c7a127de79121333e6400c2b0954ae37 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 15:55:54 -0700 Subject: [PATCH 05/17] feat(sdk): flows plugin remove and update (A1) `flows plugin remove ` drops the github: declaration, rebuilds lock order, and deletes the store directory only when nothing in the lock still references it. `flows plugin update [] [--to ]` re-resolves, prints the permissions/events/budget diff, and rewrites store/lock/flows.json only with --yes (exit 2 otherwise). Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/cli-commands.ts | 18 +- packages/sdk/src/cli.ts | 2 + packages/sdk/src/cli/add-extension.ts | 37 ++- packages/sdk/src/cli/plugin.ts | 247 +++++++++++++++++-- packages/sdk/src/plugin-lock.ts | 19 +- packages/sdk/src/plugin-store.ts | 5 + packages/sdk/tests/plugin-extension.test.ts | 102 +++++++- packages/sdk/tests/relay-cli-surface.test.ts | 8 + 8 files changed, 411 insertions(+), 27 deletions(-) diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index 23d6237d2..ad1e5aeb1 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -201,7 +201,7 @@ export const CLI_VERBS = [ }, { name: 'plugin', - description: 'Inspect the flow-extension plugins recorded in flows.lock.json', + description: 'Inspect, remove, or update the flow-extension plugins recorded in flows.lock.json', subcommands: [ { name: 'list', description: 'List installed flow extensions in composition order', options: [JSON_OPTION] }, { @@ -209,6 +209,22 @@ export const CLI_VERBS = [ description: 'Re-hash .flows/plugins against the lockfile and, unless --offline, against the pinned commit on GitHub', options: [JSON_OPTION, { flags: '--offline', description: 'Skip the GitHub re-fetch; check only the local store against the lockfile' }], }, + { + name: 'remove', + description: 'Drop a flow-extension plugin from flows.json, the lockfile, and the local store', + args: [{ name: 'name', description: 'Plugin name as recorded in the lockfile', required: true }], + options: [JSON_OPTION], + }, + { + name: 'update', + description: 'Re-resolve a flow-extension plugin, show the permissions/events/budget diff, and rewrite the lock with --yes', + args: [{ name: 'name', description: 'Plugin name; omit to update every installed flow extension', required: false }], + options: [ + JSON_OPTION, + { flags: '--to ', description: 'GitHub reference to resolve instead of the locked commit' }, + { flags: '--yes', description: 'Apply the update; without this flag the diff is printed and the lock is left unchanged' }, + ], + }, ], variants: ['plugin'], }, diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 24ac1fe6f..97274bb27 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -97,6 +97,8 @@ const USAGE = [ 'flows add ', 'flows plugin list [--json]', 'flows plugin verify [--json] [--offline]', + 'flows plugin remove [--json] ', + 'flows plugin update [--json] [--yes] [--to ] []', 'flows build [--out ] ', 'flows build --verify ', 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--no-connect] [--json]', diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts index c8e4bc9ae..8596bee39 100644 --- a/packages/sdk/src/cli/add-extension.ts +++ b/packages/sdk/src/cli/add-extension.ts @@ -30,17 +30,50 @@ export function extensionManifestOf(plugin: FetchedPlugin): { manifest: FlowExte return { manifest, manifestSha256: sha256(file.data) }; } +function eventKeys(manifest: FlowExtensionManifest): readonly string[] { + return manifest.triggers.map(t => `${t.provider} ${t.event}[${t.actions.join(',')}]`); +} + +function formatBudget(budget: FlowExtensionManifest['permissions']['budget']): string { + if (budget === undefined) return 'inherits base'; + return [budget.dollars === undefined ? '' : `$${budget.dollars}`, budget.wallclock ?? ''].filter(Boolean).join(' / ') || 'inherits base'; +} + export function describeExtension(manifest: FlowExtensionManifest): string[] { const p = manifest.permissions; return [ ` integrations: ${p.integrations.join(', ') || 'none'}; harnesses: ${p.harnesses.join(', ') || 'none'}; mcp: ${p.mcp.join(', ') || 'none'}`, - ` events: ${manifest.triggers.map(t => `${t.provider} ${t.event}[${t.actions.join(',')}]`).join('; ') || 'none'}`, + ` events: ${eventKeys(manifest).join('; ') || 'none'}`, ` hooks: ${manifest.extends.hooks.join(', ') || 'none'}; handlers: ${manifest.extends.handlers ? 'yes' : 'no'}`, ` writes (declared, unenforced): ${p.writes.join(', ') || 'none'}`, - ` budget: ${p.budget === undefined ? 'inherits base' : [p.budget.dollars === undefined ? '' : `$${p.budget.dollars}`, p.budget.wallclock ?? ''].filter(Boolean).join(' / ')}`, + ` budget: ${formatBudget(p.budget)}`, ]; } +/** Permissions / events / budget (and version/hooks) changes between two manifests. */ +export function diffExtension(before: FlowExtensionManifest, after: FlowExtensionManifest): string[] { + const lines: string[] = []; + const change = (label: string, oldValue: string, newValue: string): void => { + if (oldValue !== newValue) lines.push(` ${label}: ${oldValue} → ${newValue}`); + }; + const setDiff = (label: string, oldList: readonly string[], newList: readonly string[]): void => { + const removed = oldList.filter(x => !newList.includes(x)); + const added = newList.filter(x => !oldList.includes(x)); + if (removed.length === 0 && added.length === 0) return; + lines.push(` ${label}: ${[...removed.map(x => `-${x}`), ...added.map(x => `+${x}`)].join(', ')}`); + }; + change('version', before.version, after.version); + setDiff('integrations', before.permissions.integrations, after.permissions.integrations); + setDiff('harnesses', before.permissions.harnesses, after.permissions.harnesses); + setDiff('mcp', before.permissions.mcp, after.permissions.mcp); + setDiff('writes', before.permissions.writes, after.permissions.writes); + setDiff('events', eventKeys(before), eventKeys(after)); + setDiff('hooks', before.extends.hooks, after.extends.hooks); + change('budget', formatBudget(before.permissions.budget), formatBudget(after.permissions.budget)); + if (lines.length === 0) lines.push(' (no permissions/events/budget changes)'); + return lines; +} + export interface AddExtensionOptions { cwd?: string; fetch?: FetchLike; diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts index 045d4841a..b4ec4f17d 100644 --- a/packages/sdk/src/cli/plugin.ts +++ b/packages/sdk/src/cli/plugin.ts @@ -1,26 +1,66 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; import type { CliIo } from '../cli.js'; -import { fetchGithubPlugin, type FetchLike } from '../plugin-github.js'; -import { lockedPlugins, readPluginLock, reconcileDeclaredExtensions, type PluginLockEntry } from '../plugin-lock.js'; +import { diffExtension, extensionManifestOf } from './add-extension.js'; +import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; +import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; +import { fetchGithubPlugin, resolveGithubSha, type FetchLike } from '../plugin-github.js'; +import { + PLUGIN_LOCK_FILE, lockForDeclared, lockWithPlugin, lockedPlugins, readPluginLock, reconcileDeclaredExtensions, + writePluginLock, type PluginLock, type PluginLockEntry, +} from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; -import { pluginStoreDirectory, verifyStoredPlugin } from '../plugin-store.js'; +import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; +import { materializePlugin, pluginStoreDirectory, removeStoredPlugin, verifyStoredPlugin } from '../plugin-store.js'; export type PluginArgs = | { command: 'plugin'; sub: 'list'; json: boolean } - | { command: 'plugin'; sub: 'verify'; json: boolean; offline: boolean }; + | { command: 'plugin'; sub: 'verify'; json: boolean; offline: boolean } + | { command: 'plugin'; sub: 'remove'; json: boolean; name: string } + | { command: 'plugin'; sub: 'update'; json: boolean; yes: boolean; name: string | undefined; to: string | undefined }; -/** `flows plugin list [--json]` · `flows plugin verify [--json] [--offline]` */ +/** `flows plugin list|verify|remove|update` */ export function parsePluginArgs(args: readonly string[]): PluginArgs | undefined { const [sub, ...rest] = args; - if (sub !== 'list' && sub !== 'verify') return undefined; + if (sub === 'list' || sub === 'verify') { + let json = false; + let offline = false; + for (const arg of rest) { + if (arg === '--json' && !json) json = true; + else if (arg === '--offline' && !offline && sub === 'verify') offline = true; + else return undefined; + } + return sub === 'list' ? { command: 'plugin', sub, json } : { command: 'plugin', sub, json, offline }; + } + if (sub === 'remove') { + let json = false; + let name: string | undefined; + for (const arg of rest) { + if (arg === '--json' && !json) json = true; + else if (arg.startsWith('-') || name !== undefined) return undefined; + else name = arg; + } + return name === undefined ? undefined : { command: 'plugin', sub: 'remove', json, name }; + } + if (sub !== 'update') return undefined; let json = false; - let offline = false; - for (const arg of rest) { + let yes = false; + let to: string | undefined; + let name: string | undefined; + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]!; if (arg === '--json' && !json) json = true; - else if (arg === '--offline' && !offline && sub === 'verify') offline = true; - else return undefined; + else if (arg === '--yes' && !yes) yes = true; + else if (arg === '--to') { + const value = rest[i + 1]; + if (to !== undefined || value === undefined || value.startsWith('-')) return undefined; + to = value; + i += 1; + } else if (arg.startsWith('-') || name !== undefined) return undefined; + else name = arg; } - return sub === 'list' ? { command: 'plugin', sub, json } : { command: 'plugin', sub, json, offline }; + return { command: 'plugin', sub: 'update', json, yes, name, to }; } /** @@ -44,7 +84,14 @@ export async function verifyPlugins(root: string, options: { offline: boolean; f return results; } -export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: { cwd?: string; fetch?: FetchLike } = {}): Promise<0 | 2> { +export interface PluginCommandOptions { + cwd?: string; + fetch?: FetchLike; + now?: () => Date; + versions?: RuntimeVersions; +} + +export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: PluginCommandOptions = {}): Promise<0 | 2> { try { const root = findPluginProject(options.cwd ?? process.cwd()); if (!root) throw new PluginError('plugin_manifest_invalid', 'flows plugin requires a project with flows.json.'); @@ -55,11 +102,15 @@ export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: { for (const { entry, ref } of plugins) io.stdout(`${entry.order}. ${entry.name}@${entry.version} ${ref} sha256:${entry.digest}`); return 0; } - const results = await verifyPlugins(root, { offline: parsed.offline, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) }); - if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: results.map(r => ({ name: r.entry.name, ref: r.ref, digest: r.entry.digest, remote: r.remote })) })); return 0; } - for (const r of results) io.stdout(`OK ${r.entry.name}@${r.entry.version} ${r.ref} local digest matches lockfile; remote ${r.remote}`); - if (results.length === 0) io.stdout('No flow-extension plugins to verify.'); - return 0; + if (parsed.sub === 'verify') { + const results = await verifyPlugins(root, { offline: parsed.offline, ...(options.fetch === undefined ? {} : { fetch: options.fetch }) }); + if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: results.map(r => ({ name: r.entry.name, ref: r.ref, digest: r.entry.digest, remote: r.remote })) })); return 0; } + for (const r of results) io.stdout(`OK ${r.entry.name}@${r.entry.version} ${r.ref} local digest matches lockfile; remote ${r.remote}`); + if (results.length === 0) io.stdout('No flow-extension plugins to verify.'); + return 0; + } + if (parsed.sub === 'remove') return await removePlugin(root, parsed, io); + return await updatePlugins(root, parsed, io, options); } catch (error) { const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); if (parsed.json) io.stdout(JSON.stringify({ ok: false, code: refusal.code, message: refusal.message })); @@ -67,3 +118,165 @@ export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: { return 2; } } + +function readFlowsConfig(root: string): { path: string; config: Record; plugins: string[] } { + const path = join(root, 'flows.json'); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(path, 'utf8')); } + catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed) + || ((parsed as { plugins?: unknown }).plugins !== undefined + && (!Array.isArray((parsed as { plugins?: unknown }).plugins) + || !(parsed as { plugins: unknown[] }).plugins.every(p => typeof p === 'string')))) { + throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); + } + const config = parsed as Record; + return { path, config, plugins: [...((config.plugins as string[] | undefined) ?? [])] }; +} + +function writeFlowsPlugins(path: string, config: Record, plugins: readonly string[]): void { + config.plugins = [...plugins]; + writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`); +} + +function storedManifest(directory: string): FlowExtensionManifest { + let raw: string; + try { raw = readFileSync(join(directory, 'flows-plugin.json'), 'utf8'); } + catch { throw new PluginError('plugin_source_drift', `${directory}: flows-plugin.json is missing.`); } + let input: unknown; + try { input = JSON.parse(raw); } + catch { throw new PluginError('plugin_manifest_invalid', `${directory}: flows-plugin.json is unreadable or invalid JSON.`); } + return validateFlowExtensionManifest(input); +} + +function storeReferenced(lock: PluginLock, name: string, digest: string): boolean { + return lock.plugins.some(p => p.name === name && p.digest === digest); +} + +async function dropUnreferencedStore(root: string, name: string, digest: string, lock: PluginLock): Promise { + if (storeReferenced(lock, name, digest)) return; + await removeStoredPlugin(pluginStoreDirectory(root, name, digest)); +} + +async function removePlugin(root: string, parsed: Extract, io: CliIo): Promise<0 | 2> { + const locked = lockedPlugins(readPluginLock(root)); + const match = locked.find(p => p.entry.name === parsed.name); + if (match === undefined) throw new PluginError('plugin_manifest_invalid', `No flow-extension plugin named ${parsed.name}.`); + const { path, config, plugins } = readFlowsConfig(root); + const nextDeclared = plugins.filter(ref => ref !== match.ref); + const nextLock = lockForDeclared(readPluginLock(root), nextDeclared); + writeFlowsPlugins(path, config, nextDeclared); + writePluginLock(root, nextLock); + await dropUnreferencedStore(root, match.entry.name, match.entry.digest, nextLock); + if (parsed.json) { + io.stdout(JSON.stringify({ ok: true, removed: { name: match.entry.name, ref: match.ref, digest: match.entry.digest } })); + return 0; + } + io.stdout(`Removed ${match.entry.name}@${match.entry.version} ${match.ref}`); + return 0; +} + +function replaceDeclaredRef(plugins: readonly string[], oldRef: string, newRef: string): string[] { + const without = plugins.filter(ref => ref !== oldRef); + if (without.includes(newRef)) return without; + const at = plugins.indexOf(oldRef); + const next = [...without]; + next.splice(at === -1 ? next.length : at, 0, newRef); + return next; +} + +async function updatePlugins( + root: string, + parsed: Extract, + io: CliIo, + options: PluginCommandOptions, +): Promise<0 | 2> { + if (parsed.to !== undefined && parsed.name === undefined) { + throw new PluginError('plugin_manifest_invalid', 'flows plugin update --to requires a plugin name.'); + } + const locked = lockedPlugins(readPluginLock(root)); + const targets = parsed.name === undefined ? locked : locked.filter(p => p.entry.name === parsed.name); + if (parsed.name !== undefined && targets.length === 0) { + throw new PluginError('plugin_manifest_invalid', `No flow-extension plugin named ${parsed.name}.`); + } + if (targets.length === 0) { + if (parsed.json) { io.stdout(JSON.stringify({ ok: true, plugins: [] })); return 0; } + io.stdout('No flow-extension plugins to update.'); + return 0; + } + + type Plan = { + current: (typeof locked)[number]; + ref: string; + digest: string; + manifest: FlowExtensionManifest; + manifestSha256: string; + source: { host: 'github'; owner: string; repo: string; sha: string; path: string }; + files: readonly { path: string; data: Uint8Array }[]; + diff: string[]; + changed: boolean; + }; + const plans: Plan[] = []; + for (const current of targets) { + const requested = parsed.to === undefined ? { ...current.source } : parsePluginSource(parsed.to); + const source = await resolveGithubSha(requested, options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + if (manifest.name !== current.entry.name) { + throw new PluginError('plugin_manifest_invalid', `Update of ${current.entry.name} resolved to plugin ${manifest.name}; remove it and add the new name instead.`); + } + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const directory = pluginStoreDirectory(root, current.entry.name, current.entry.digest); + const before = storedManifest(directory); + const ref = canonicalPluginRef(source); + const changed = ref !== current.ref || fetched.digest !== current.entry.digest || manifestSha256 !== current.entry.manifestSha256; + plans.push({ + current, ref, digest: fetched.digest, manifest, manifestSha256, + source: { host: 'github', owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }, + files: fetched.files, diff: diffExtension(before, manifest), changed, + }); + } + + const pending = plans.filter(p => p.changed); + const summary = plans.map(p => ({ + name: p.current.entry.name, from: p.current.ref, to: p.ref, digest: p.digest, changed: p.changed, diff: p.diff, + })); + if (!parsed.json) { + for (const plan of plans) { + if (!plan.changed) { + io.stdout(`${plan.current.entry.name} is already at ${plan.ref} sha256:${plan.digest}`); + continue; + } + io.stdout(`Update ${plan.current.entry.name} ${plan.current.ref} → ${plan.ref}`); + io.stdout(` digest sha256:${plan.current.entry.digest} → sha256:${plan.digest}`); + for (const line of plan.diff) io.stdout(line); + } + } + if (pending.length === 0) { + if (parsed.json) io.stdout(JSON.stringify({ ok: true, plugins: summary })); + return 0; + } + if (!parsed.yes) throw new PluginError('plugin_manifest_invalid', 'Re-run with --yes to apply this update.'); + + let declared = readFlowsConfig(root).plugins; + let lock = readPluginLock(root); + const applied: { name: string; ref: string; digest: string }[] = []; + for (const plan of pending) { + const { directory, digest } = await materializePlugin(root, plan.manifest.name, plan.files); + if (digest !== plan.digest) throw new PluginError('plugin_source_drift', 'Materialized digest differs from the fetched digest.'); + declared = replaceDeclaredRef(declared, plan.current.ref, plan.ref); + lock = lockWithPlugin(lock, declared, { + name: plan.manifest.name, version: plan.manifest.version, source: plan.source, + digest, manifestSha256: plan.manifestSha256, + resolvedAt: (options.now ?? (() => new Date()))().toISOString(), + }); + const { path, config } = readFlowsConfig(root); + writeFlowsPlugins(path, config, declared); + writePluginLock(root, lock); + await dropUnreferencedStore(root, plan.current.entry.name, plan.current.entry.digest, lock); + applied.push({ name: plan.manifest.name, ref: plan.ref, digest }); + } + if (parsed.json) io.stdout(JSON.stringify({ ok: true, plugins: applied })); + else for (const item of applied) io.stdout(`Updated ${item.name} ${item.ref} sha256:${item.digest} recorded in flows.json and ${PLUGIN_LOCK_FILE}`); + return 0; +} diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index 94bbc208d..3628d0f62 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -79,14 +79,12 @@ export function writePluginLock(root: string, lock: PluginLock): void { /** * Rebuild the entry list in `flows.json.plugins` order: `order` is derived from * the declaration list, never stored independently, so the two cannot disagree. + * Lock entries that are no longer declared are dropped; a declaration with no + * matching lock entry is a refusal. */ -export function lockWithPlugin( - lock: PluginLock, declared: readonly string[], - entry: Omit, -): PluginLock { +export function lockForDeclared(lock: PluginLock, declared: readonly string[]): PluginLock { const byRef = new Map(lock.plugins.map(p => [canonicalPluginRef({ ...p.source, ref: p.source.sha }), p])); - byRef.set(canonicalPluginRef({ ...entry.source, ref: entry.source.sha }), { ...entry, kind: 'flow-extension', order: 0 }); - const plugins = declared.filter(ref => ref.startsWith('github:')).map((ref, index) => { + const plugins = declared.filter(isGithubPluginRef).map((ref, index) => { const found = byRef.get(ref); if (found === undefined) return invalid(`flows.json declares ${ref} but the lockfile has no entry for it; run flows add ${ref}.`); return Object.freeze({ ...found, order: index + 1 }); @@ -94,6 +92,15 @@ export function lockWithPlugin( return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze(plugins) }); } +export function lockWithPlugin( + lock: PluginLock, declared: readonly string[], + entry: Omit, +): PluginLock { + const byRef = new Map(lock.plugins.map(p => [canonicalPluginRef({ ...p.source, ref: p.source.sha }), p])); + byRef.set(canonicalPluginRef({ ...entry.source, ref: entry.source.sha }), { ...entry, kind: 'flow-extension', order: 0 }); + return lockForDeclared({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze([...byRef.values()]) }, declared); +} + /** Lock entries paired with their declared reference, in declaration order. */ export function lockedPlugins(lock: PluginLock): readonly { ref: string; entry: PluginLockEntry; source: PluginSourceRef }[] { return lock.plugins.map(entry => { diff --git a/packages/sdk/src/plugin-store.ts b/packages/sdk/src/plugin-store.ts index 61b5275f8..04af2518c 100644 --- a/packages/sdk/src/plugin-store.ts +++ b/packages/sdk/src/plugin-store.ts @@ -81,6 +81,11 @@ export async function verifyStoredPlugin(directory: string, expectedDigest: stri await rejectExtras(directory, '', new Set([...paths, 'manifest.json']), drift); } +/** Drop a materialized plugin directory. Missing is a no-op. */ +export async function removeStoredPlugin(directory: string): Promise { + await rm(directory, { recursive: true, force: true }); +} + async function rejectExtras(root: string, prefix: string, paths: Set, drift: (m: string) => never): Promise { for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { const path = prefix + entry.name; diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts index 75e2f6daf..481713f01 100644 --- a/packages/sdk/tests/plugin-extension.test.ts +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -4,7 +4,7 @@ import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { addPlugin } from '../src/cli/add.js'; import { addExtensionPlugin } from '../src/cli/add-extension.js'; -import { runPluginCommand, verifyPlugins } from '../src/cli/plugin.js'; +import { parsePluginArgs, runPluginCommand, verifyPlugins } from '../src/cli/plugin.js'; import { runCli } from '../src/cli.js'; import { validateFlowExtensionManifest } from '../src/flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha } from '../src/plugin-github.js'; @@ -265,6 +265,106 @@ describe('flows plugin list / verify', () => { }); }); +describe('flows plugin remove / update', () => { + async function installed(entries: FakeEntry[] = babysitter) { + const gh = github(entries); const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + p.messages.length = 0; + return { gh, p, digest: readPluginLock(p.cwd).plugins[0]!.digest }; + } + it('drops the declaration, rebuilds lock order, and deletes the store directory', async () => { + const sitter = babysitter.map(e => e.path.endsWith('flows-plugin.json') + ? { ...e, path: e.path.replace('examples/babysitter', 'examples/sitter'), data: Buffer.from(JSON.stringify({ ...manifestJson, name: 'sitter' })) } + : { ...e, path: e.path.replace('examples/babysitter', 'examples/sitter') }); + const gh = fakeGithub({ + 'AgentWorkforce/flows': { + refs: { 'feat/babysitter-v2': SHA_A, main: SHA_A }, + commits: { [SHA_A]: { entries: [...babysitter, ...sitter] } }, + }, + }); + const p = project(); + expect(await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + const sitterRef = `github:AgentWorkforce/flows@${SHA_A}#examples/sitter`; + expect(await addExtensionPlugin(sitterRef, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions })).toBe(0); + const before = readPluginLock(p.cwd); + expect(before.plugins.map(e => [e.order, e.name])).toEqual([[1, 'babysitter'], [2, 'sitter']]); + const babysitterDir = pluginStoreDirectory(p.cwd, 'babysitter', before.plugins[0]!.digest); + const sitterDir = pluginStoreDirectory(p.cwd, 'sitter', before.plugins[1]!.digest); + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'remove', json: false, name: 'babysitter' }, p.io, { cwd: p.cwd })).toBe(0); + expect(p.text()).toContain(`Removed babysitter@0.1.0 ${REF}`); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([sitterRef]); + const after = readPluginLock(p.cwd); + expect(after.plugins.map(e => [e.order, e.name])).toEqual([[1, 'sitter']]); + expect(existsSync(babysitterDir)).toBe(false); + expect(existsSync(sitterDir)).toBe(true); + }); + it('refuses to remove a name that is not installed', async () => { + const { p } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'remove', json: false, name: 'nope' }, p.io, { cwd: p.cwd })).toBe(2); + expect(p.text()).toContain('REFUSED [plugin_manifest_invalid] No flow-extension plugin named nope.'); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + }); + it('shows the permissions/events/budget diff and writes nothing without --yes', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ + ...m, + version: '0.2.0', + permissions: { ...(m.permissions as object), writes: ['github:pull_request:comment', 'github:issue:comment'], budget: { dollars: 12, wallclock: '1h' } }, + }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + updated.repos['AgentWorkforce/flows']!.refs.main = SHA_B; + const to = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: false, yes: false, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(2); + expect(p.text()).toContain(`Update babysitter ${REF} → ${to}`); + expect(p.text()).toContain('version: 0.1.0 → 0.2.0'); + expect(p.text()).toContain('+github:issue:comment'); + expect(p.text()).toContain('budget: $8 / 45m → $12 / 1h'); + expect(p.text()).toContain('REFUSED [plugin_manifest_invalid] Re-run with --yes to apply this update.'); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([REF]); + expect(readPluginLock(p.cwd).plugins[0]!.digest).toBe(digest); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', digest))).toBe(true); + }); + it('applies --to with --yes, rewrites store/lock/flows.json, and drops the old store', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ ...m, version: '0.2.0', description: 'next' }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + updated.repos['AgentWorkforce/flows']!.refs.main = SHA_B; + const to = `github:AgentWorkforce/flows@main#examples/babysitter`; + const canonical = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: false, yes: true, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(0); + const lock = readPluginLock(p.cwd); + expect(lock.plugins).toHaveLength(1); + expect(lock.plugins[0]).toMatchObject({ name: 'babysitter', version: '0.2.0', order: 1, source: { sha: SHA_B } }); + expect(lock.plugins[0]!.digest).not.toBe(digest); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins).toEqual([canonical]); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', digest))).toBe(false); + expect(existsSync(pluginStoreDirectory(p.cwd, 'babysitter', lock.plugins[0]!.digest))).toBe(true); + expect(p.text()).toContain(`Updated babysitter ${canonical}`); + }); + it('reports already-at when re-resolving the locked commit', async () => { + const { gh, p, digest } = await installed(); + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: true, yes: false, name: undefined, to: undefined }, p.io, { + cwd: p.cwd, fetch: gh.fetch, now, versions, + })).toBe(0); + expect(JSON.parse(p.text())).toMatchObject({ ok: true, plugins: [{ name: 'babysitter', changed: false, digest }] }); + }); + it('parses the new subcommands and refuses a malformed invocation', () => { + expect(parsePluginArgs(['remove', 'babysitter'])).toEqual({ command: 'plugin', sub: 'remove', json: false, name: 'babysitter' }); + expect(parsePluginArgs(['update', '--yes'])).toEqual({ command: 'plugin', sub: 'update', json: false, yes: true, name: undefined, to: undefined }); + expect(parsePluginArgs(['update', 'babysitter', '--to', 'github:o/r@main#x', '--yes', '--json'])) + .toEqual({ command: 'plugin', sub: 'update', json: true, yes: true, name: 'babysitter', to: 'github:o/r@main#x' }); + expect(parsePluginArgs(['remove'])).toBeUndefined(); + expect(parsePluginArgs(['update', '--to'])).toBeUndefined(); + expect(parsePluginArgs(['update', '--yes', '--yes'])).toBeUndefined(); + }); +}); + describe('legacy helper plugins are untouched', () => { it('never contacts GitHub for a helper name and leaves the helper path to npm', async () => { const gh = github(); const p = project(); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 97caf8b3d..ff1297304 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -169,6 +169,14 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'plugin', argv: ['plugin', 'list', '--json'], variant: 'plugin' }, { verb: 'plugin', argv: ['plugin', 'verify'], variant: 'plugin' }, { verb: 'plugin', argv: ['plugin', 'verify', '--json', '--offline'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'remove', 'babysitter'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'remove', '--json', 'babysitter'], variant: 'plugin' }, + { verb: 'plugin', argv: ['plugin', 'update', '--yes'], variant: 'plugin' }, + { + verb: 'plugin', + argv: ['plugin', 'update', 'babysitter', '--to', 'github:o/r@main#path', '--yes', '--json'], + variant: 'plugin', + }, { verb: 'undeploy', argv: ['undeploy', 'dep_123'], variant: 'undeploy' }, { verb: 'undeploy', argv: ['undeploy', '--json', 'dep_123'], variant: 'undeploy' }, { verb: 'unschedule', argv: ['unschedule', 'sched_123'], variant: 'unschedule' }, From f5bec9d0596323e3e9a9b17cd941f9c38910d4ed Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 16:00:54 -0700 Subject: [PATCH 06/17] feat(sdk): seal flow-extension plugins in bundle lockfile v2 (A2) Sealed bundle lockfile.json is now the v2 plugin lock. Materialized extension files are copied under plugins// and hashed by the existing envelope. npm pins move to package-lock.json. Authored-root metadata records extensions: [{name,digest,ref}] for journal provenance. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/authored-root.ts | 22 ++++++- packages/sdk/src/authored-source-authority.ts | 6 ++ packages/sdk/src/bundle-extensions.ts | 59 +++++++++++++++++++ packages/sdk/src/bundle-typescript.ts | 2 +- packages/sdk/src/cli/build.ts | 9 ++- packages/sdk/src/plugin-lock.ts | 5 +- packages/sdk/src/plugin-store.ts | 10 ++++ packages/sdk/tests/authored-root.test.ts | 21 +++++++ packages/sdk/tests/bundle.test.ts | 43 ++++++++++++++ 9 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 packages/sdk/src/bundle-extensions.ts diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index eb8adc78c..30b525dc3 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -21,6 +21,12 @@ import { isSurfaceCompletionReason } from './authored-step-output.js'; const ROOT_KIND = 'relayflows.authored-root.v1'; +export interface AuthoredRootExtension { + readonly name: string; + readonly digest: string; + readonly ref: string; +} + export interface AuthoredRootMetadata { readonly kind: typeof ROOT_KIND; readonly flowName: string; @@ -28,6 +34,8 @@ export interface AuthoredRootMetadata { readonly sourceSha256: string; readonly surface: SurfaceModuleAuthority; readonly sources: readonly AuthoredRootSourceAuthority[]; + /** Plugin-level provenance in lock order; empty when the project declares none. */ + readonly extensions: readonly AuthoredRootExtension[]; readonly localAgentStream?: string; readonly inputPresent: boolean; readonly input?: unknown; @@ -68,6 +76,9 @@ export async function executeDurableAuthoredFlow( sourceSha256: sha256(source), surface: loaded.surfaceAuthority, sources: Object.freeze(sources), + extensions: Object.freeze((loaded.extensions ?? []).map(extension => Object.freeze({ + name: extension.name, digest: extension.digest, ref: extension.ref, + }))), ...(options.localAgentStream === undefined ? {} : { localAgentStream: options.localAgentStream }), inputPresent: input !== undefined, ...(input === undefined ? {} : { input: jsonSnapshot(input, 'authored root input') }), @@ -172,7 +183,7 @@ export async function readAuthoredRootMetadata( if (!isRootMetadata(value)) { throw new Error('authored root journal has malformed authority metadata'); } - return value; + return Object.freeze({ ...value, extensions: value.extensions ?? [] }); } async function driveRoot( @@ -409,10 +420,19 @@ function isRootMetadata(value: unknown): value is AuthoredRootMetadata { && typeof surface.version === 'string' && /^[a-f0-9]{64}$/.test(surface.packageSha256 ?? '') && /^[a-f0-9]{64}$/.test(surface.runtimeSha256 ?? '') && Array.isArray(sources) && sources.length > 0 && sources.every(isRootSourceAuthority) + && (root.extensions === undefined || (Array.isArray(root.extensions) && root.extensions.every(isRootExtension))) && (root.localAgentStream === undefined || /^local-agent-[a-f0-9-]+$/.test(root.localAgentStream)); } +function isRootExtension(value: unknown): value is AuthoredRootExtension { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const extension = value as Partial; + return typeof extension.name === 'string' && extension.name.length > 0 + && /^[a-f0-9]{64}$/.test(extension.digest ?? '') + && typeof extension.ref === 'string' && extension.ref.startsWith('github:'); +} + function isRootSourceAuthority(value: unknown): value is AuthoredRootSourceAuthority { if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; const source = value as Partial; diff --git a/packages/sdk/src/authored-source-authority.ts b/packages/sdk/src/authored-source-authority.ts index ec045465e..a543ce0c9 100644 --- a/packages/sdk/src/authored-source-authority.ts +++ b/packages/sdk/src/authored-source-authority.ts @@ -35,6 +35,12 @@ export async function loadPinnedAuthoredSource(metadata: AuthoredRootMetadata, i if (canonicalize(loadedSources) !== canonicalize(metadata.sources)) { throw new Error('authored root declared source graph authority mismatch'); } + const loadedExtensions = (loaded.extensions ?? []).map(extension => ({ + name: extension.name, digest: extension.digest, ref: extension.ref, + })); + if (canonicalize(loadedExtensions) !== canonicalize(metadata.extensions ?? [])) { + throw new Error('authored root declared extension authority mismatch'); + } return loaded; } diff --git a/packages/sdk/src/bundle-extensions.ts b/packages/sdk/src/bundle-extensions.ts new file mode 100644 index 000000000..9bf92e308 --- /dev/null +++ b/packages/sdk/src/bundle-extensions.ts @@ -0,0 +1,59 @@ +import { readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import type { BundleFile } from './bundle.js'; +import { sha256 } from './bundle.js'; +import { findPluginProject } from './plugin-loader.js'; +import { parsePluginLock, readPluginLock, reconcileDeclaredExtensions, type PluginLock } from './plugin-lock.js'; +import { PluginError } from './plugin-manifest.js'; +import { pluginStoreDirectory, readStoredPluginFiles } from './plugin-store.js'; + +const EMPTY_LOCK: PluginLock = Object.freeze({ version: 2, plugins: Object.freeze([]) }); + +/** + * Plugin provenance and bytes to put in a sealed bundle: `lockfile.json` is + * the same v2 lock the project records, and each extension's materialized + * files land under `plugins//…`. No project / no github: plugins → + * `{ version: 2, plugins: [] }` and no files. Fail closed on lock/store drift. + */ +export async function collectBundleExtensions(flowPath: string): Promise<{ lock: PluginLock; files: BundleFile[] }> { + const root = findPluginProject(dirname(flowPath)); + if (root === undefined) return { lock: EMPTY_LOCK, files: [] }; + const locked = reconcileDeclaredExtensions(root); + const files: BundleFile[] = []; + for (const { entry } of locked) { + const stored = await readStoredPluginFiles(pluginStoreDirectory(root, entry.name, entry.digest), entry.digest); + for (const file of stored) files.push({ path: `plugins/${entry.name}/${file.path}`, data: file.data }); + } + return { lock: readPluginLock(root), files }; +} + +/** + * After `verifyBundle` has hashed every payload file, check that `lockfile.json` + * is a v2 plugin lock (or a legacy v1 `{version:1, adapters}` with no plugins) + * and that each locked extension's store digest matches `plugins//manifest.json`. + */ +export async function verifyBundlePluginLock(bundle: string): Promise { + let parsed: unknown; + try { parsed = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); } + catch { throw new Error('lockfile.json: not valid JSON'); } + if (isLegacyV1Lock(parsed)) return; + let lock: PluginLock; + try { lock = parsePluginLock(parsed); } + catch (error) { + throw new Error(error instanceof PluginError ? error.message : 'lockfile.json: not a plugin lock'); + } + for (const entry of lock.plugins) { + let manifest: Buffer; + try { manifest = await readFile(join(bundle, 'plugins', entry.name, 'manifest.json')); } + catch { throw new Error(`lockfile.json: plugin ${entry.name} is missing plugins/${entry.name}/manifest.json`); } + if (sha256(manifest) !== entry.digest) { + throw new Error(`lockfile.json: plugin ${entry.name} digest does not match plugins/${entry.name}/manifest.json`); + } + } +} + +function isLegacyV1Lock(value: unknown): boolean { + return typeof value === 'object' && value !== null && !Array.isArray(value) + && (value as { version?: unknown }).version === 1 + && Array.isArray((value as { adapters?: unknown }).adapters); +} diff --git a/packages/sdk/src/bundle-typescript.ts b/packages/sdk/src/bundle-typescript.ts index 8164c1a07..4014287bf 100644 --- a/packages/sdk/src/bundle-typescript.ts +++ b/packages/sdk/src/bundle-typescript.ts @@ -60,7 +60,7 @@ process.stdout.write(JSON.stringify({ spec, authored })); '--no-compile-autoload-bunfig', '--outfile', executable, input], directory); const files: BundleFile[] = [ { path: 'flow', data: await readFile(executable) }, - { path: 'lockfile.json', data: canonicalize(lock) }, + { path: 'package-lock.json', data: canonicalize(lock) }, ]; return { spec, authored: inspected.authored === true, compiler, files }; } finally { await rm(staging, { recursive: true, force: true }); } diff --git a/packages/sdk/src/cli/build.ts b/packages/sdk/src/cli/build.ts index 30ffa4988..61425cda7 100644 --- a/packages/sdk/src/cli/build.ts +++ b/packages/sdk/src/cli/build.ts @@ -2,6 +2,7 @@ import { readFile, lstat } from 'node:fs/promises'; import { dirname, extname, join, resolve } from 'node:path'; import { parse } from 'yaml'; import { sealBundle, verifyBundle, type BundleFile } from '../bundle.js'; +import { collectBundleExtensions, verifyBundlePluginLock } from '../bundle-extensions.js'; import { canonicalize } from '../canonical.js'; import { compileSpec, toKernelSpec } from '../compile.js'; import { preflight } from '../preflight.js'; @@ -60,6 +61,7 @@ export async function runBuild(args: BuildArgs, io: CliIo): Promise<0 | 2> { try { if (args.verify) { const digest = `sha256:${await verifyBundle(args.value)}`; + await verifyBundlePluginLock(args.value); // `--json` is declared on the verb, not on one of its forms: a verify // under it emits the same single object a `--json` consumer parses. if (args.json) io.stdout(JSON.stringify({ ok: true, verified: true, bundle: args.value, digest })); @@ -107,7 +109,7 @@ export async function buildFlow(path: string, out: string, warn: (line: string) files.push(...result.files); } else if (['.yaml', '.yml'].includes(extname(input))) { authoring = compileSpec(parse(await readFile(input, 'utf8'))); - files.push({ path: 'lockfile.json', data: canonicalize({ version: 1, adapters: [] }) }); + files.push({ path: 'lockfile.json', data: canonicalize({ version: 2, plugins: [] }) }); } else throw new Error('build expects a .yaml, .yml, or .ts flow'); // The current preflight API reports uncollected environment facts as @@ -148,7 +150,10 @@ export async function buildFlow(path: string, out: string, warn: (line: string) ...(authored ? { dynamicSteps: 'exported spec declaration checked; body was not executed during build' } : {}) }, }) }, ); - return sealBundle({ name: authoring.name ?? 'flow', out, files, + const extensions = await collectBundleExtensions(input); + const bundled = files.filter(file => file.path !== 'lockfile.json'); + bundled.push({ path: 'lockfile.json', data: canonicalize(extensions.lock) }, ...extensions.files); + return sealBundle({ name: authoring.name ?? 'flow', out, files: bundled, repo: await repositoryRoot(directory), warn }); } diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index 3628d0f62..000aec90e 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -7,9 +7,8 @@ import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, ty * `flows.lock.json` — the project's plugin provenance. `flows.json.plugins` * says *what* is declared; the lockfile says exactly which bytes that meant: * the commit, the content digest, the manifest hash, and the order the - * operator declared. Version 2 because the sealed bundle's `lockfile.json` - * is version 1 and the two will converge on this shape when bundles carry - * plugins (RFC-0001 decision 14). + * operator declared. Version 2 is also the sealed bundle `lockfile.json` + * shape when the bundle carries plugins (RFC-0001 decision 14). */ export const PLUGIN_LOCK_FILE = 'flows.lock.json'; export const PLUGIN_LOCK_VERSION = 2; diff --git a/packages/sdk/src/plugin-store.ts b/packages/sdk/src/plugin-store.ts index 04af2518c..8ff7d232f 100644 --- a/packages/sdk/src/plugin-store.ts +++ b/packages/sdk/src/plugin-store.ts @@ -81,6 +81,16 @@ export async function verifyStoredPlugin(directory: string, expectedDigest: stri await rejectExtras(directory, '', new Set([...paths, 'manifest.json']), drift); } +/** Re-verify, then return every stored file including the payload `manifest.json`. */ +export async function readStoredPluginFiles(directory: string, expectedDigest: string): Promise { + await verifyStoredPlugin(directory, expectedDigest); + const manifest = await regularFile(directory, 'manifest.json'); + const entries = JSON.parse(manifest.toString('utf8')) as { path: string }[]; + const files = [{ path: 'manifest.json', data: manifest }]; + for (const entry of entries) files.push({ path: entry.path, data: await regularFile(directory, entry.path) }); + return files; +} + /** Drop a materialized plugin directory. Missing is a no-op. */ export async function removeStoredPlugin(directory: string): Promise { await rm(directory, { recursive: true, force: true }); diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index bee894829..acc8367da 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -159,9 +159,28 @@ describe('durable authored root', () => { flowPath: loaded.sourcePath, input: { topic: 'relay' }, surface, }); expect(metadata.sourceSha256).toMatch(/^[a-f0-9]{64}$/u); + expect(metadata.extensions).toEqual([]); expect(journal.peer.completions).toEqual([{ attempt: 1, reason: 'success' }]); }); + it('journals plugin digests from composed extensions', async () => { + const loaded = await fixture(); + const extension = { + name: 'babysitter', + digest: 'c'.repeat(64), + ref: `github:AgentWorkforce/flows@${'a'.repeat(40)}#examples/babysitter`, + }; + const journal = new RootJournal(); + await executeDurableAuthoredFlow( + { ...loaded, extensions: [extension as LoadedAuthoredFlow['extensions'][number]] }, + journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'with-plugin' }, + ); + const step = (journal.starts[0]!.spec as { steps: Array<{ instruction: string }> }).steps[0]!; + const metadata = JSON.parse(step.instruction) as AuthoredRootMetadata; + expect(metadata.extensions).toEqual([extension]); + }); + it('reconciles a lost start acknowledgement from the durable root result', async () => { const loaded = await fixture(); const journal = new RootJournal(); @@ -364,6 +383,7 @@ async function fixture( return { sourcePath, handle, getDefinition, surfaceAuthority: surface, graph: [{ path: sourcePath, handle, getDefinition, surfaceAuthority: surface, use: [] }], + extensions: [], }; } @@ -394,6 +414,7 @@ function spawnedEntry(loaded: LoadedAuthoredFlow): Record { sourceSha256: '76fd521c5bda4f37b3c69a6ae3c5a97f0a2ba53d3d8b09d9cc9709f809f5a5a2', surface, }], + extensions: [], inputPresent: false, }; return { diff --git a/packages/sdk/tests/bundle.test.ts b/packages/sdk/tests/bundle.test.ts index 1c517eea7..8a61478b5 100644 --- a/packages/sdk/tests/bundle.test.ts +++ b/packages/sdk/tests/bundle.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest'; +import { existsSync } from 'node:fs'; import { mkdtemp, readFile, readdir, writeFile, rm, mkdir, symlink, stat, chmod, rename } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; @@ -6,7 +7,10 @@ import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; import { canonicalize } from '../src/canonical.js'; import { sealBundle, sha256, verifyBundle } from '../src/bundle.js'; +import { verifyBundlePluginLock } from '../src/bundle-extensions.js'; +import { addExtensionPlugin } from '../src/cli/add-extension.js'; import { buildFlow } from '../src/cli/build.js'; +import { SHA_A, entriesFromDirectory, fakeGithub } from './fake-github.js'; const sdk = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const repo = resolve(sdk, '../..'); @@ -218,6 +222,45 @@ describe('immutable bundles', () => { expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); }); + it('writes lockfile.json v2 with no plugins for a project that declares none', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'hello.yaml'), await readFile(join(repo, 'testdata/hello-deterministic.flow.yaml'))); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + expect(JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8'))).toEqual({ plugins: [], version: 2 }); + await verifyBundlePluginLock(bundle); + }); + + it('seals materialized flow-extension files under plugins// and verifies them', async () => { + const cwd = await temp(); + const fixture = join(repo, 'testdata/plugins/extension-babysitter'); + const gh = fakeGithub({ + 'AgentWorkforce/flows': { + refs: { main: SHA_A }, + commits: { [SHA_A]: { entries: entriesFromDirectory(fixture, 'examples/babysitter') } }, + }, + }); + await writeFile(join(cwd, 'flows.json'), JSON.stringify({})); + const io = { stdout: () => {}, stderr: () => {} }; + expect(await addExtensionPlugin(`github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`, io, { + cwd, fetch: gh.fetch, now: () => new Date('2026-09-20T12:00:00Z'), versions: { sdk: '2.0.22', surface: '2.0.22' }, + })).toBe(0); + await writeFile(join(cwd, 'hello.yaml'), await readFile(join(repo, 'testdata/hello-deterministic.flow.yaml'))); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + const lock = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); + expect(lock.version).toBe(2); + expect(lock.plugins).toHaveLength(1); + expect(lock.plugins[0]).toMatchObject({ name: 'babysitter', kind: 'flow-extension', order: 1 }); + expect(existsSync(join(bundle, 'plugins/babysitter/flows-plugin.json'))).toBe(true); + expect(existsSync(join(bundle, 'plugins/babysitter/babysitter.flow.ts'))).toBe(true); + expect(existsSync(join(bundle, 'plugins/babysitter/manifest.json'))).toBe(true); + expect(sha256(await readFile(join(bundle, 'plugins/babysitter/manifest.json')))).toBe(lock.plugins[0].digest); + expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); + await verifyBundlePluginLock(bundle); + const tampered = join(bundle, 'plugins/babysitter/babysitter.flow.ts'); + await writeFile(tampered, `${await readFile(tampered, 'utf8')}\n// tampered\n`); + await expect(verifyBundle(bundle)).rejects.toThrow('babysitter.flow.ts'); + }); + it('refuses build-provable CLI resolution errors without environment probes', async () => { const cwd = await temp(); await writeFile(join(cwd, 'invalid.yaml'), JSON.stringify({ version: '0.1.0', name: 'invalid', steps: [ From 49d16cfe72b3c89716282174dc1eb55c086d9520 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 16:09:43 -0700 Subject: [PATCH 07/17] feat(surface,sdk): compose f.hook implementations in lock order (A3) FlowHeader gains version and hooks. Ctx.hook AND-composes installed plugin implementations as journaled child steps; a recorded verdict is replayed and the closure is not re-run. Software Garden declares the three hook points and calls merge-gate before opening a PASSED PR. Co-Authored-By: Claude Opus 5 (1M context) --- docs/SURFACE.md | 20 ++-- .../software-factory/software-factory.flow.ts | 21 +++- packages/sdk/src/authored-flow-executor.ts | 29 +++++ packages/sdk/src/authored-flow-loader.ts | 6 +- packages/sdk/src/authored-hooks.ts | 106 ++++++++++++++++++ packages/sdk/src/authored-node-entry.ts | 1 + packages/sdk/src/authored-root.ts | 1 + packages/sdk/src/cli/check-triggers.ts | 1 + packages/sdk/src/cli/check.ts | 2 + packages/sdk/src/flow-extension-compat.ts | 7 +- packages/sdk/src/flow-extension-loader.ts | 39 ++++++- packages/sdk/tests/authored-hooks.test.ts | 88 +++++++++++++++ .../sdk/tests/flow-extension-compose.test.ts | 33 +++++- packages/surface/src/context.ts | 6 + packages/surface/src/flow.ts | 46 ++++++-- packages/surface/tests/flow.test.ts | 16 +++ packages/ts-plugin/src/rules/header-keys.ts | 2 +- 17 files changed, 390 insertions(+), 34 deletions(-) create mode 100644 packages/sdk/src/authored-hooks.ts create mode 100644 packages/sdk/tests/authored-hooks.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 0e780f837..a91296eb8 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -604,16 +604,18 @@ and the authored root) composes the project's extensions onto the base flow declaration and the lockfile must agree; the store is re-hashed against the lock's digest and the manifest bytes against its manifest hash — nothing under `.flows/plugins` is read as code before that passes; the manifest is validated -and its `compat` checked against the runtime and the base flow (the base has no -version field yet, so only `*` is satisfiable; a budget ceiling above the base -is `plugin_incompatible`); only then is the entry imported, its handlers -checked against the manifest's declared triggers (an entry cannot subscribe to -more than it declared), and appended **after** the base's own handlers in -lockfile order. Nothing replaces, reorders, or widens a base handler, and the -base's definition object is untouched. `flows check` prints one `EXTENSION` +and its `compat` checked against the runtime and the base flow (`FlowHeader.version` +is matched when present; without it only `*` is satisfiable; a budget ceiling +above the base is `plugin_incompatible`); only then is the entry imported, its +handlers checked against the manifest's declared triggers (an entry cannot +subscribe to more than it declared), and appended **after** the base's own +handlers in lockfile order. Named `hooks` exports are matched to +`extends.hooks` and to the base header's `hooks` list; `f.hook` AND-composes +them in lock order. Nothing replaces, reorders, or widens a base handler, and +the base's definition object is untouched. `flows check` prints one `EXTENSION` line per composed extension. Not composed by this release, and refused with -`plugin_unsupported` rather than ignored: hooks, an entry `use:` header, -schedule triggers, and gates; a generic `webhook(...)` handler is refused as +`plugin_unsupported` rather than ignored: an entry `use:` header, schedule +triggers, and gates; a generic `webhook(...)` handler is refused as undeclared. Cloud deploy and hosted runs refuse a project with extensions (`unsupported_source`) because the deploy body carries one source file and would silently lose them. Handler bodies still execute nowhere (#301); what diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index 43c37a8f4..90312e072 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -29,7 +29,11 @@ const WORK = ".relayflow"; // test result, the exit code does. Skips honestly when there is nothing to run. const TEST = 'if [ -f package.json ] && node -e \'p=require("./package.json");process.exit(p.scripts&&p.scripts.test?0:1)\'; then npm ci --no-audit --no-fund && npm test; else echo "no test script; skipping"; fi'; -export default flow("software-factory", { budget: { dollars: 10, wallclock: "1h" } }, async (f, input) => { +export default flow("software-factory", { + version: "2.0.22", + hooks: ["pre-implement", "post-review", "merge-gate"], + budget: { dollars: 10, wallclock: "1h" }, +}, async (f, input) => { const { issue } = input; if (!issue?.title?.trim()) { // Parked, not canceled: a body cannot declare a kernel outcome, and the @@ -68,8 +72,21 @@ export default flow("software-factory", { budget: { dollars: 10, wallcloc await f.run("git push --set-upstream origin HEAD"); // Deterministic step, not an agent decision: the PR is opened either way, - // but a blocked review opens it as a draft with the findings attached. + // but a blocked review or a false merge-gate opens it as a draft. if (verdict.trim() === "PASSED") { + const origin = (await f.run("git remote get-url origin")).trim(); + const headSha = (await f.run("git rev-parse HEAD")).trim(); + const matched = /github\.com[:/]([^/]+)\/([^/.]+)/.exec(origin); + const allowed = await f.hook("merge-gate", { + owner: matched?.[1] ?? "", + repo: matched?.[2] ?? "", + headSha, + }); + if (!allowed) { + await f.run(`{ cat ${WORK}/summary.md; printf '\\n\\n## merge-gate: blocked\\n\\n'; } > ${WORK}/pr-body.md`); + await f.run(`gh pr create --draft --title ${shellWord(`[blocked] ${title}`)} --body-file ${WORK}/pr-body.md`); + return f.done("step_failed"); + } await f.run(`gh pr create --title ${shellWord(title)} --body-file ${WORK}/summary.md`); return f.done("success"); } diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 965ad117b..9d961e540 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -44,6 +44,8 @@ import { } from './authored-flow-operation.js'; import { AuthoredFlowLifecycle } from './authored-flow-lifecycle.js'; import { JournalClient } from './journal-client.js'; +import { createHookEvaluator } from './authored-hooks.js'; +import type { LoadedFlowExtension } from './flow-extension-loader.js'; import type { CompletionReason as ProtocolCompletionReason, RunCompletionReason as ProtocolRunCompletionReason, @@ -148,6 +150,8 @@ export interface ExecuteAuthoredFlowOptions { readonly localAgentStream?: string; /** Durable kernel root that owns this body's child admission identities. */ readonly rootRunId?: string; + /** Installed flow-extension plugins, in lock order, so `f.hook` can AND-compose them. */ + readonly extensions?: readonly LoadedFlowExtension[]; } export async function executeAuthoredFlow( @@ -349,6 +353,14 @@ export async function executeAuthoredFlow( )); } + const evaluateHook = createHookEvaluator({ + journal, + ...(options.rootRunId === undefined ? {} : { rootRunId: options.rootRunId }), + flowName: definition.name, + declared: definition.header.hooks ?? [], + extensions: options.extensions ?? [], + }); + const context: Ctx = { ...createHelpers((call: HelperCall): Step => { const verb = `${call.provider}.${call.verb}`; @@ -477,6 +489,23 @@ export async function executeAuthoredFlow( assertOperationAllowed('dispatch', definition.name, requestedCompletion); throw unsupportedVerb('dispatch'); }, + hook(name, input) { + assertOperationAllowed('hook', definition.name, requestedCompletion); + const id = `hook-${nextStep++}`; + const snapshot = snapshotJsonValue(input, 'f.hook input'); + return trackStep(authoredSteps, new AuthoredFlowOperation( + id, 'hook', + () => assertOperationAllowed('hook', definition.name, requestedCompletion), + async () => { + const verdict = await evaluateHook(id, name, snapshot, context); + const record = { hook: name, step: id, verdict: verdict ? 'pass' : 'fail' }; + const literal = `'${JSON.stringify(record).replaceAll("'", "'\\''")}'`; + await observeStep(id, 'deterministic', () => lowerDeterministic(id, `printf '%s' ${literal}`, false), options.onProgress); + return verdict; + }, + lifecycle, + )); + }, done(reason) { if (!isSurfaceFlowCompletionReason(reason)) { throw new AuthoredFlowExecutionError( diff --git a/packages/sdk/src/authored-flow-loader.ts b/packages/sdk/src/authored-flow-loader.ts index 3118815be..5a7db01a4 100644 --- a/packages/sdk/src/authored-flow-loader.ts +++ b/packages/sdk/src/authored-flow-loader.ts @@ -10,7 +10,7 @@ import { type FlowHandle, } from './authored-flow.js'; import { canonicalize } from './canonical.js'; -import { composeDefinition, loadFlowExtensions, type LoadedFlowExtension } from './flow-extension-loader.js'; +import { composeDefinition, loadFlowExtensions, parseHooksExport, type ImportedFlow, type LoadedFlowExtension } from './flow-extension-loader.js'; import type { RuntimeVersions } from './flow-extension-compat.js'; export class AuthoredFlowLoadError extends Error { @@ -145,7 +145,7 @@ export async function loadAuthoredFlow(path: string, options: LoadAuthoredFlowOp surfaceAuthority: root.surfaceAuthority, graph: Object.freeze(graph), extensions }); } -async function importAuthoredFlow(path: string): Promise> { +async function importAuthoredFlow(path: string): Promise> { const absolutePath = resolve(path); try { accessSync(absolutePath, constants.R_OK); @@ -171,7 +171,7 @@ async function importAuthoredFlow(path: string): Promise Promise { + let recorded: Promise> | undefined; + + async function load(): Promise> { + const verdicts = new Map(); + if (options.rootRunId === undefined) return verdicts; + let offset = 0; + for (;;) { + const page = await options.journal.streamRead(options.rootRunId, HOOK_STREAM, offset, 1000); + for (const message of page.messages) { + const raw = (message as { message?: unknown }).message ?? message; + if (typeof raw !== 'object' || raw === null) continue; + const record = raw as HookRecord; + if (typeof record.hook !== 'string' || typeof record.step !== 'string') continue; + if (record.verdict !== 'pass' && record.verdict !== 'fail' && record.verdict !== 'noop') continue; + if (record.plugin !== null && typeof record.plugin !== 'string') continue; + verdicts.set(recordKey(record), record); + } + if (page.messages.length === 0 || page.next_offset <= offset) break; + offset = page.next_offset; + } + return verdicts; + } + + async function lookup(key: string): Promise { + if (options.rootRunId === undefined) return undefined; + recorded ??= load(); + return (await recorded).get(key); + } + + async function append(record: HookRecord): Promise { + if (options.rootRunId === undefined) return; + recorded ??= load(); + await options.journal.streamAppend(options.rootRunId, HOOK_STREAM, record); + (await recorded).set(recordKey(record), record); + } + + return async (id, name, input, context) => { + if (!options.declared.includes(name)) { + throw new AuthoredFlowExecutionError( + 'unsupported_verb', + `hook "${name}" is not declared by flow "${options.flowName}"`, + ); + } + const impls = options.extensions.filter(extension => extension.hooks[name] !== undefined); + if (impls.length === 0) { + const existing = await lookup(`${id}:noop`); + const record = existing ?? { hook: name, step: id, plugin: null, verdict: 'noop' as const }; + if (existing === undefined) await append(record); + return true; + } + for (const extension of impls) { + const key = `${id}:${extension.name}`; + let record = await lookup(key); + if (record === undefined) { + let verdict = false; + let because: string | undefined; + try { + verdict = await extension.hooks[name]!(context, input) === true; + } catch (error) { + verdict = false; + because = error instanceof Error ? error.message : String(error); + } + record = { + hook: name, step: id, plugin: extension.name, + verdict: verdict ? 'pass' : 'fail', + ...(because === undefined ? {} : { because }), + }; + await append(record); + } + if (record.verdict === 'fail') return false; + } + return true; + }; +} diff --git a/packages/sdk/src/authored-node-entry.ts b/packages/sdk/src/authored-node-entry.ts index 367840ebf..1909ea5f1 100644 --- a/packages/sdk/src/authored-node-entry.ts +++ b/packages/sdk/src/authored-node-entry.ts @@ -72,6 +72,7 @@ try { request.metadata.inputPresent ? request.metadata.input : undefined, { getDefinition: loaded.getDefinition, dataDir: request.dataDir, flowPath: request.metadata.flowPath, rootRunId: request.rootRunId, + extensions: loaded.extensions, localAgentStream: request.localAgentStream, signal: controller.signal, onProgress: event => send({ type: 'progress', event }), onWait: event => send({ type: 'wait', event }), diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 30b525dc3..0b6513042 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -214,6 +214,7 @@ async function driveRoot( flowPath: metadata.flowPath, localAgentStream: options.localAgentStream, rootRunId: dispatch.run_id, + extensions: loaded.extensions, ...options.lifecycle, signal: callerSignal === undefined ? rootSignal diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index 55c27a66d..9742dd397 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -45,6 +45,7 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const extensions = (loaded.extensions ?? []).map(extension => ({ name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, handlers: extension.handlers.length, + hooks: Object.keys(extension.hooks ?? {}), })); return { loaded, diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 8a6c195a3..4b72e3cce 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -74,6 +74,8 @@ export interface ExtensionInspection { digest: string; /** How many `.on()` handlers it appends after the base flow's own. */ handlers: number; + /** Hook names this extension implements, in manifest order. */ + hooks?: readonly string[]; } export interface ScheduleInspection { diff --git a/packages/sdk/src/flow-extension-compat.ts b/packages/sdk/src/flow-extension-compat.ts index 53af83400..6c3101ccc 100644 --- a/packages/sdk/src/flow-extension-compat.ts +++ b/packages/sdk/src/flow-extension-compat.ts @@ -19,11 +19,8 @@ export function assertCompatible(manifest: FlowExtensionManifest, versions: Runt } /** - * `compat.base` against the flow being extended. The surface's `FlowHeader` - * carries no version field, so a base flow has no version to satisfy a range - * with: only `*` can be met today, and anything narrower is refused rather - * than assumed. When the header grows a `version`, this is the one place to - * read it. + * `compat.base` against the flow being extended. `FlowHeader.version` is + * optional: a base without it matches only `"*"`. */ export function assertBaseCompatible(manifest: FlowExtensionManifest, base: { readonly name: string; readonly version?: string }): void { const entry = manifest.compat.base.find(b => b.name === base.name); diff --git a/packages/sdk/src/flow-extension-loader.ts b/packages/sdk/src/flow-extension-loader.ts index 5a7f6c3a3..9dc845be2 100644 --- a/packages/sdk/src/flow-extension-loader.ts +++ b/packages/sdk/src/flow-extension-loader.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; +import type { Ctx } from '@relayflows/surface'; import type { AuthoredFlowDefinition, FlowHandle } from './authored-flow.js'; import { sha256 } from './bundle.js'; import { assertBaseCompatible, assertCompatible, runtimeVersions, type RuntimeVersions } from './flow-extension-compat.js'; @@ -20,7 +21,7 @@ import { pluginStoreDirectory, verifyStoredPlugin } from './plugin-store.js'; * `.flows/plugins` is read as code before this passes; * 3. the manifest is validated, its compat checked against the runtime and * the base flow, and anything this slice does not compose (hooks, `use`, - * schedule triggers, non-provider webhooks) is refused; + * schedule triggers, non-provider webhooks, gates) is refused; * 4. only then is the entry imported, and its handlers are checked against * the manifest's declared triggers — an entry cannot subscribe to more * than it declared. @@ -29,6 +30,8 @@ import { pluginStoreDirectory, verifyStoredPlugin } from './plugin-store.js'; */ type TriggerHandler = AuthoredFlowDefinition['handlers'][number]; +export type FlowHook = (f: Ctx, input: unknown) => Promise; + export interface LoadedFlowExtension { readonly name: string; readonly version: string; @@ -41,12 +44,14 @@ export interface LoadedFlowExtension { /** Bound to the surface copy the entry itself imported; the base's accessor cannot see this handle's WeakMap entry. */ readonly getDefinition: ImportedFlow['getDefinition']; readonly handlers: readonly TriggerHandler[]; + readonly hooks: Readonly>; } export interface ImportedFlow { readonly handle: FlowHandle; readonly getDefinition: (handle: FlowHandle) => AuthoredFlowDefinition; readonly surfaceAuthority: Authority; + readonly hooks: Readonly>; } export interface LoadFlowExtensionsOptions { @@ -101,8 +106,7 @@ async function loadOne( const manifest = validateFlowExtensionManifest(input); if (manifest.name !== lock.name || manifest.version !== lock.version) throw new PluginError('plugin_source_drift', `${ref}: manifest names ${manifest.name}@${manifest.version}, lockfile has ${lock.name}@${lock.version}.`); assertCompatible(manifest, options.versions ?? runtimeVersions()); - assertBaseCompatible(manifest, { name: base.definition.name }); - if (manifest.extends.hooks.length > 0) unsupported(manifest.name, `hooks (${manifest.extends.hooks.join(', ')})`); + assertBaseCompatible(manifest, { name: base.definition.name, version: base.definition.header.version }); const baseBudget = base.definition.header.budget; const ceiling = manifest.permissions.budget; if (ceiling?.dollars !== undefined && typeof baseBudget === 'object' && baseBudget.dollars !== undefined && ceiling.dollars > baseBudget.dollars) { @@ -125,12 +129,41 @@ async function loadOne( throw new PluginError('plugin_manifest_invalid', `${manifest.name}: ${manifest.entry} declares handlers but extends.handlers is false.`); } definition.handlers.forEach((handler, index) => assertDeclaredSubscription(manifest.name, manifest, handler, index)); + const hooks = imported.hooks; + const exported = Object.keys(hooks).sort(); + const declared = [...manifest.extends.hooks].sort(); + if (exported.join('\0') !== declared.join('\0')) { + throw new PluginError('plugin_manifest_invalid', `${manifest.name}: extends.hooks [${manifest.extends.hooks.join(', ')}] does not match exported hooks [${exported.join(', ')}].`); + } + const baseHooks = base.definition.header.hooks ?? []; + for (const hook of manifest.extends.hooks) { + if (!baseHooks.includes(hook)) { + throw new PluginError('plugin_incompatible', `${manifest.name}: hook ${hook} is not declared by the base flow.`); + } + } return Object.freeze({ name: manifest.name, version: manifest.version, ref, digest: lock.digest, directory, entryPath, manifest, handle: imported.handle, getDefinition: imported.getDefinition, handlers: Object.freeze([...definition.handlers]), + hooks, }); } +export function parseHooksExport(module: Record, name: string): Readonly> { + const exported = module['hooks']; + if (exported === undefined) return Object.freeze({}); + if (typeof exported !== 'object' || exported === null || Array.isArray(exported)) { + throw new PluginError('plugin_manifest_invalid', `${name}: hooks export must be a record of functions.`); + } + const hooks: Record = {}; + for (const [key, value] of Object.entries(exported)) { + if (typeof value !== 'function') { + throw new PluginError('plugin_manifest_invalid', `${name}: hooks.${key} is not a function.`); + } + hooks[key] = value as FlowHook; + } + return Object.freeze(hooks); +} + /** Extensions declared by the project that owns `flowPath`, verified and loaded in lock order; empty when none are declared. */ export async function loadFlowExtensions( flowPath: string, diff --git a/packages/sdk/tests/authored-hooks.test.ts b/packages/sdk/tests/authored-hooks.test.ts new file mode 100644 index 000000000..7c46d4bf3 --- /dev/null +++ b/packages/sdk/tests/authored-hooks.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; +import type { Ctx } from '@relayflows/surface'; +import { createHookEvaluator } from '../src/authored-hooks.js'; +import type { LoadedFlowExtension } from '../src/flow-extension-loader.js'; +import type { JournalClient } from '../src/journal-client.js'; + +function fakeJournal() { + const streams = new Map(); + const journal = { + async streamRead(_run: string, stream: string, offset: number) { + const messages = (streams.get(stream) ?? []).slice(offset); + return { messages: messages.map(message => ({ message })), next_offset: (streams.get(stream) ?? []).length }; + }, + async streamAppend(_run: string, stream: string, message: unknown) { + const list = streams.get(stream) ?? []; + list.push(message); + streams.set(stream, list); + return { offset: list.length }; + }, + } as unknown as JournalClient; + return { journal, streams }; +} + +function extension(name: string, impl: (f: Ctx, input: unknown) => Promise): LoadedFlowExtension { + return { name, hooks: { 'merge-gate': impl } } as LoadedFlowExtension; +} + +describe('hook AND composition', () => { + const ctx = {} as Ctx; + + it('is a journaled no-op returning true when nothing implements the hook', async () => { + const { journal, streams } = fakeJournal(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], extensions: [], + }); + await expect(evaluate('hook-1', 'merge-gate', {}, ctx)).resolves.toBe(true); + expect(streams.get('hooks')).toEqual([{ hook: 'merge-gate', step: 'hook-1', plugin: null, verdict: 'noop' }]); + }); + + it('runs implementations in lock order and AND-composes, stopping at the first false', async () => { + const { journal, streams } = fakeJournal(); + const order: string[] = []; + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [ + extension('first', async () => { order.push('first'); return true; }), + extension('second', async () => { order.push('second'); return false; }), + extension('third', async () => { order.push('third'); return true; }), + ], + }); + await expect(evaluate('hook-1', 'merge-gate', { owner: 'o' }, ctx)).resolves.toBe(false); + expect(order).toEqual(['first', 'second']); + expect(streams.get('hooks')).toEqual([ + { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'pass' }, + { hook: 'merge-gate', step: 'hook-1', plugin: 'second', verdict: 'fail' }, + ]); + }); + + it('replays a recorded verdict and does not re-run the closure', async () => { + const { journal, streams } = fakeJournal(); + streams.set('hooks', [ + { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'fail', because: 'held' }, + ]); + let calls = 0; + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [extension('first', async () => { calls += 1; return true; })], + }); + await expect(evaluate('hook-1', 'merge-gate', {}, ctx)).resolves.toBe(false); + expect(calls).toBe(0); + expect(streams.get('hooks')).toHaveLength(1); + }); + + it('refuses a hook the base header does not declare', async () => { + const { journal } = fakeJournal(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], extensions: [], + }); + await expect(evaluate('hook-1', 'nope', {}, ctx)).rejects.toMatchObject({ + code: 'unsupported_verb', + message: expect.stringContaining('hook "nope" is not declared'), + }); + }); +}); diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 8538cff82..6ae59889f 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -123,7 +123,7 @@ describe('composing flow extensions onto a base flow', () => { await install(p); const { report } = await checkAuthoredTriggers(p.flow); expect(report.ok).toBe(true); - expect(report.extensions).toEqual([{ name: 'babysitter', version: '0.1.0', ref: REF, digest: expect.stringMatching(/^[0-9a-f]{64}$/), handlers: 8 }]); + expect(report.extensions).toEqual([{ name: 'babysitter', version: '0.1.0', ref: REF, digest: expect.stringMatching(/^[0-9a-f]{64}$/), handlers: 8, hooks: [] }]); expect(report.requirements?.integrations.map(i => i.provider)).toContain('github'); expect(await runCli(['check', p.flow], p.io)).toBe(0); expect(p.text()).toContain(`EXTENSION babysitter@0.1.0 ${REF} sha256:`); @@ -157,12 +157,41 @@ describe('composition fails closed', () => { ['a base flow it does not extend', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'other-flow', version: '*' }] } }), 'plugin_incompatible', 'not "software-factory"'], ['a base version range the unversioned base cannot satisfy', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^1.0.0' }] } }), 'plugin_incompatible', 'declares no version'], ['a budget ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 11 } } }), 'plugin_incompatible', 'above the base flow'], - ['hooks, which this release does not compose', (m: Record) => ({ ...m, extends: { handlers: true, hooks: ['merge-gate'] } }), 'plugin_unsupported', 'hooks (merge-gate)'], ])('refuses %s', async (_, patch, code, message) => { const p = project(); await install(p, variant(patch)); await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ code, message: expect.stringContaining(message) }); }); + const HOOK_ENTRY = "import { flow, github } from '@relayflows/surface';\nexport const hooks = { 'merge-gate': async () => true };\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); });\n"; + it('refuses a hook export that the manifest does not declare', async () => { + const p = project(); + await install(p, variant(m => m, HOOK_ENTRY)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ + code: 'plugin_manifest_invalid', message: expect.stringContaining('does not match exported hooks'), + }); + }); + it('refuses a hook the base header does not declare', async () => { + const p = project(); + await install(p, variant(m => ({ ...m, extends: { handlers: true, hooks: ['merge-gate'] } }), HOOK_ENTRY)); + await expect(loadAuthoredFlow(p.flow, { versions })).rejects.toMatchObject({ + code: 'plugin_incompatible', message: expect.stringContaining('hook merge-gate is not declared'), + }); + }); + it('composes a declared hook when the base header names it and reads header.version for compat', async () => { + const p = project(` + import { flow, github } from '@relayflows/surface'; + export default flow('software-factory', { version: '2.0.22', hooks: ['merge-gate'], budget: { dollars: 10, wallclock: '1h' } }, async f => { f.done('success'); }) + .on(github.issues({ action: 'opened' }), async f => { f.done('success'); }); + `); + await install(p, variant(m => ({ + ...m, + extends: { handlers: true, hooks: ['merge-gate'] }, + compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^2.0.0' }] }, + }), HOOK_ENTRY)); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + expect(Object.keys(loaded.extensions[0]!.hooks)).toEqual(['merge-gate']); + expect(loaded.getDefinition(loaded.handle).header).toMatchObject({ version: '2.0.22', hooks: ['merge-gate'] }); + }); it.each([ ['an entry subscribing beyond its manifest', undefined, "import { flow, github } from '@relayflows/surface';\nexport default flow('babysitter', async f => { f.done('success'); }).on(github.pull_request('opened'), async f => { f.done('success'); }).on(github.pull_request('labeled'), async f => { f.done('success'); });\n", diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index 38b48ce74..a9820ae76 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -72,6 +72,12 @@ export interface Ctx extends Helpers { */ human(question: string, options: { to: string }): Step; dispatch(flow: string, input: unknown): Promise; + /** + * Run every installed implementation of a named hook in lock order and + * AND-compose the booleans. With no implementations this is a journaled + * no-op that returns true. A name must appear in the flow header's `hooks`. + */ + hook(name: string, input: unknown): Step; done(reason: FlowCompletionReason): void; cloud: CloudHelper; memory: MemoryHelper; diff --git a/packages/surface/src/flow.ts b/packages/surface/src/flow.ts index fd06adeb5..bb0b84cd1 100644 --- a/packages/surface/src/flow.ts +++ b/packages/surface/src/flow.ts @@ -9,6 +9,10 @@ import { schedule } from "./schedule.js"; export interface FlowHeader { /** Relative paths to reusable authored flows composed by this body. */ use?: string[]; + /** Semver of this flow; plugins' `compat.base` ranges match against it. */ + version?: string; + /** Named hook points this body calls via `f.hook`; plugins may implement them. */ + hooks?: string[]; identity?: string; memory?: { script?: boolean; agent?: boolean }; budget?: string | { tokens?: number; dollars?: number; wallclock?: string }; @@ -20,6 +24,8 @@ export type FlowBody = (f: Ctx, input: Input) => Promise; export interface ReadonlyFlowHeader { readonly use?: readonly string[]; + readonly version?: string; + readonly hooks?: readonly string[]; readonly identity?: string; readonly memory?: Readonly<{ script?: boolean; agent?: boolean }>; readonly budget?: string | Readonly<{ tokens?: number; dollars?: number; wallclock?: string }>; @@ -163,15 +169,19 @@ function isStoredDefinition( && Object.isFrozen(value); } +const HEADER_FIELDS = [ + "use", + "version", + "hooks", + "identity", + "memory", + "budget", + "tools", + "workspace", +] as const; + function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { - const unknownFields = Object.keys(header).filter((field) => ![ - "use", - "identity", - "memory", - "budget", - "tools", - "workspace", - ].includes(field)); + const unknownFields = Object.keys(header).filter((field) => !(HEADER_FIELDS as readonly string[]).includes(field)); if (unknownFields.length > 0) { throw new TypeError(`flow header has unknown fields: ${unknownFields.join(", ")}`); } @@ -192,6 +202,8 @@ function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { }); return Object.freeze({ ...(header.use === undefined ? {} : { use: Object.freeze([...header.use]) }), + ...(header.version === undefined ? {} : { version: header.version }), + ...(header.hooks === undefined ? {} : { hooks: Object.freeze([...header.hooks]) }), ...(header.identity === undefined ? {} : { identity: header.identity }), ...(memory === undefined ? {} : { memory }), ...(header.budget === undefined ? {} : { budget: typeof header.budget === "string" ? header.budget : Object.freeze({ ...header.budget }) }), @@ -205,10 +217,14 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl assertHeaderObject(value, at); assertKnownKeys( value, - ["use", "identity", "memory", "budget", "tools", "workspace"], + HEADER_FIELDS, at, ); assertOptionalString(value, "identity", at); + assertOptionalString(value, "version", at); + if (value.version !== undefined && (value.version as string).trim().length === 0) { + throw new TypeError(`${at}.version: expected a nonempty version string`); + } if (value.budget !== undefined && typeof value.budget !== "string") { assertHeaderObject(value.budget, `${at}.budget`); assertKnownKeys(value.budget, ["tokens", "dollars", "wallclock"], `${at}.budget`); @@ -219,6 +235,18 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl } assertOptionalString(value, "workspace", at); assertOptionalStringArray(value, "use", at); + assertOptionalStringArray(value, "hooks", at); + if (value.hooks !== undefined) { + const hooks = value.hooks as string[]; + if (hooks.some((item) => item.trim().length === 0) || new Set(hooks).size !== hooks.length) { + throw new TypeError(`${at}.hooks: expected unique nonempty names`); + } + for (const hook of hooks) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(hook)) { + throw new TypeError(`${at}.hooks: expected kebab-case names`); + } + } + } if (value.use !== undefined) { for (const path of value.use as string[]) { if (!/^(?:\.\/|\.\.\/).+\.flow\.ts$/.test(path) || /[?#\\\\]/.test(path)) { diff --git a/packages/surface/tests/flow.test.ts b/packages/surface/tests/flow.test.ts index b919104f0..4147e17b1 100644 --- a/packages/surface/tests/flow.test.ts +++ b/packages/surface/tests/flow.test.ts @@ -62,6 +62,22 @@ describe("flow", () => { expect(Object.isFrozen(getFlowDefinition(definition).header.tools?.mcp)).toBe(true); }); + it("freezes version and hooks on the header", () => { + const hooks = ["pre-implement", "merge-gate"]; + const handle = flow("software-factory", { version: "2.0.22", hooks, budget: { dollars: 10 } }, async () => undefined); + hooks.push("later"); + expect(getFlowDefinition(handle).header).toMatchObject({ version: "2.0.22", hooks: ["pre-implement", "merge-gate"] }); + expect(Object.isFrozen(getFlowDefinition(handle).header.hooks)).toBe(true); + }); + + it.each([ + [{ hooks: ["MergeGate"] }, "header.hooks: expected kebab-case names"], + [{ hooks: ["merge-gate", "merge-gate"] }, "header.hooks: expected unique nonempty names"], + [{ version: "" }, "header.version: expected a nonempty version string"], + ])("refuses malformed version/hooks: %j", (header, message) => { + expect(() => flow("software-factory", header as FlowHeader, async () => undefined)).toThrow(message); + }); + it("validates raw header keys and nested values before cloning", () => { const invalidHeaders: { value: unknown; message: string }[] = [ { diff --git a/packages/ts-plugin/src/rules/header-keys.ts b/packages/ts-plugin/src/rules/header-keys.ts index 2b20eddbe..90af84766 100644 --- a/packages/ts-plugin/src/rules/header-keys.ts +++ b/packages/ts-plugin/src/rules/header-keys.ts @@ -3,7 +3,7 @@ import { DIAGNOSTICS, DIAGNOSTIC_SOURCE } from "../diagnostics"; // Mirrors assertFlowHeader in surface/src/flow.ts. SDK parity fixtures pin // these three closed lists without importing or executing author code in tsserver. -const HEADER_KEYS = ["identity", "memory", "budget", "tools", "workspace"]; +const HEADER_KEYS = ["identity", "memory", "budget", "tools", "workspace", "version", "hooks"]; const NESTED_KEYS: Record = { memory: ["script", "agent"], tools: ["relayfile", "mcp", "slack"], From 83e850e5d7e5592dabda476792e8f7792b144770 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 16:15:42 -0700 Subject: [PATCH 08/17] feat(sdk): send composed extensions on hosted deploy (A4-A6) Replace the unsupported_source refusal with an extensions[] deploy/run body (2 MB cap, UTF-8 or base64 files). --plugin is send-only. flows check prints the composed hook table. GitHub pull_request.ready_for_review / labeled / unlabeled stay unroutable: the surface registry is generated from the relayfile adapter catalog and cannot be grown from this repo. Co-Authored-By: Claude Opus 5 (1M context) --- docs/CLOUD.md | 11 ++ docs/SURFACE.md | 18 ++- examples/README.md | 8 ++ packages/sdk/src/cli-commands.ts | 1 + packages/sdk/src/cli.ts | 12 +- packages/sdk/src/cli/check-triggers.ts | 6 + packages/sdk/src/cli/check.ts | 9 ++ packages/sdk/src/cli/cloud-deploy.ts | 12 +- packages/sdk/src/cloud-deploy.ts | 19 ++- packages/sdk/src/cloud-run.ts | 17 ++- packages/sdk/src/flow-extension-submit.ts | 118 ++++++++++++++++++ packages/sdk/tests/cloud-deploy.test.ts | 1 + .../sdk/tests/flow-extension-compose.test.ts | 7 ++ packages/sdk/tests/relay-cli-surface.test.ts | 2 +- 14 files changed, 213 insertions(+), 28 deletions(-) create mode 100644 packages/sdk/src/flow-extension-submit.ts diff --git a/docs/CLOUD.md b/docs/CLOUD.md index 575405efb..c48de401a 100644 --- a/docs/CLOUD.md +++ b/docs/CLOUD.md @@ -43,6 +43,17 @@ An authored `.flow.ts` takes `--input` exactly as a local direct run does (an existing JSON file, otherwise inline JSON), and travels as one self-contained source with its pinned Surface authority. +Flow-extension plugins declared in `flows.json` (and extra `--plugin ` on `flows deploy`) travel in the deploy/run body as `extensions[]`: +name, version, canonical ref, digest, manifest, and the plugin files (UTF-8 +or base64). The extensions field is capped at 2 MB separately from the 256 KB +source cap. Cloud must materialize them at `.flows/plugins/@sha256:/` +before the hosted CLI loads the source; until that Cloud slice lands, a +deployment that includes plugins is accepted by this CLI but not yet executed +as a composed graph on the hosted runner. Private repositories are +unsupported. `permissions.writes` is a reviewed declaration, unenforced +until gate 8. + ## Code sync ```sh diff --git a/docs/SURFACE.md b/docs/SURFACE.md index a91296eb8..ded2d041e 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -581,6 +581,8 @@ the base flows it extends), and the same mandatory `preflight`. Validation is flows add github:/@# # or https://github.com///tree// flows plugin list [--json] flows plugin verify [--json] [--offline] +flows plugin remove [--json] +flows plugin update [--json] [--yes] [--to ] [] ``` `flows add` resolves the branch, tag, or commit to a 40-hex sha through @@ -616,11 +618,17 @@ the base's definition object is untouched. `flows check` prints one `EXTENSION` line per composed extension. Not composed by this release, and refused with `plugin_unsupported` rather than ignored: an entry `use:` header, schedule triggers, and gates; a generic `webhook(...)` handler is refused as -undeclared. Cloud deploy and hosted runs refuse a project with extensions -(`unsupported_source`) because the deploy body carries one source file and -would silently lose them. Handler bodies still execute nowhere (#301); what -composition changes today is the declared trigger set that `flows check`, -requirements, and future dispatch read. +undeclared. Cloud deploy and hosted runs send composed extensions in the request body +(`extensions[]`, 2 MB cap, `--plugin` is send-only). Handler bodies still +execute nowhere (#301); what composition changes today is the declared +trigger set that `flows check`, requirements, and future dispatch read. + +GitHub `pull_request.ready_for_review`, `pull_request.labeled`, and +`pull_request.unlabeled` are **not** in the surface registry. The registry is +generated from the pinned relayfile adapter mappings (`scripts/generate-triggers.mjs`); +this repo cannot add those actions without an adapter-package change. A +Babysitter manifest that declares them is refused `plugin_event_unroutable` +until that upstream catalog grows. ## 4. Build: the immutable bundle diff --git a/examples/README.md b/examples/README.md index efc86de66..961b5fc09 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,14 @@ flow straight from this repo, shows its steps, and asks you to connect whatever Or from a checkout: `flows deploy --repo --on --approver `. +Install a plugin onto a base flow (Babysitter on Software Garden): + +```text +flows add github:AgentWorkforce/flows@#examples/babysitter +``` + +The plugin is recorded in `flows.json` / `flows.lock.json` and composed at load time. Hosted deploy accepts `--plugin ` as a send-only extra. Private repositories are unsupported. `permissions.writes` is declared, not enforced. + ## Gallery status **3 of 4 gallery entries pass; one is blocked.** The blocked entry fails diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index ad1e5aeb1..0bf944748 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -155,6 +155,7 @@ export const CLI_VERBS = [ { flags: '--agents ', description: 'Agent harnesses to allow, as claude[,codex]' }, { flags: '--name ', description: 'Name for the hosted listener' }, { flags: '--draft', description: 'Create the listener without activating it' }, + { flags: '--plugin ', description: 'Send-only GitHub flow-extension ref; repeatable. Does not write flows.json' }, NO_CONNECT_OPTION, JSON_OPTION, ], diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 97274bb27..60569857f 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -101,7 +101,7 @@ const USAGE = [ 'flows plugin update [--json] [--yes] [--to ] []', 'flows build [--out ] ', 'flows build --verify ', - 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--no-connect] [--json]', + 'flows deploy --repo --on [:key=value,...] [--on ...] --approver [--agents claude[,codex]] [--name ] [--draft] [--plugin ] [--no-connect] [--json]', 'flows deployments [--json]', 'flows undeploy [--json] ', 'flows schedule [--cron "" | --every ] [--tz ] [--input ] [--name ] [--no-connect] [--json]', @@ -948,7 +948,15 @@ function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { io.stdout(`SCHEDULE handler ${schedule.handler} ${declared} -> flows.tick schedule_id ${schedule.scheduleId} [${local}]`); } for (const extension of report.extensions ?? []) { - io.stdout(`EXTENSION ${extension.name}@${extension.version} ${extension.ref} sha256:${extension.digest} -> ${extension.handlers} handler(s) composed after the base flow`); + const hookList = (extension.hooks ?? []).length === 0 ? '' : `, hooks: ${extension.hooks!.join(', ')}`; + io.stdout(`EXTENSION ${extension.name}@${extension.version} ${extension.ref} sha256:${extension.digest} -> ${extension.handlers} handler(s) composed after the base flow${hookList}`); + } + if (report.hooks !== undefined) { + io.stdout(`HOOKS declared: ${report.hooks.declared.join(', ') || '(none)'}`); + for (const row of report.hooks.implementations) io.stdout(`HOOK ${row.hook} <- ${row.plugin}`); + for (const name of report.hooks.declared) { + if (!report.hooks.implementations.some(row => row.hook === name)) io.stdout(`HOOK ${name} <- (none)`); + } } for (const resolution of report.resolutions) { const config = resolution.source === 'project' && report.projectConfigPath !== undefined diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index 9742dd397..4222b557e 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -47,6 +47,11 @@ export async function checkAuthoredTriggers(path: string): Promise<{ handlers: extension.handlers.length, hooks: Object.keys(extension.hooks ?? {}), })); + const declaredHooks = definition.header.hooks ?? []; + const implementations = (loaded.extensions ?? []).flatMap(extension => + Object.keys(extension.hooks ?? {}).map(hook => ({ hook, plugin: extension.name }))); + const hooks = declaredHooks.length > 0 || implementations.length > 0 + ? { declared: declaredHooks, implementations } : undefined; return { loaded, report: { @@ -55,6 +60,7 @@ export async function checkAuthoredTriggers(path: string): Promise<{ path, gates: [], resolutions: [], diagnostics, ...(schedules.length === 0 ? {} : { schedules }), ...(extensions.length === 0 ? {} : { extensions }), + ...(hooks === undefined ? {} : { hooks }), requirements: flowRequirements(definition, { projectCli: config.cli }), ...(config.path === undefined ? {} : { projectConfigPath: config.path }), }, diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 4b72e3cce..1ead84e21 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -63,6 +63,8 @@ export interface CheckReport { requirements?: FlowRequirements; /** Schema-2 flow extensions composed onto the authored flow, in lock order (`flow-extension-loader.ts`). */ extensions?: ExtensionInspection[]; + /** Base hook points and which plugins implement them. */ + hooks?: HookInspection; diagnostics: Array; } @@ -78,6 +80,13 @@ export interface ExtensionInspection { hooks?: readonly string[]; } +export interface HookInspection { + /** Names the base flow header declares. */ + declared: readonly string[]; + /** Plugin implementations in lock order. */ + implementations: readonly { hook: string; plugin: string }[]; +} + export interface ScheduleInspection { /** Position among the flow's handlers, so two identical declarations stay distinct. */ handler: number; diff --git a/packages/sdk/src/cli/cloud-deploy.ts b/packages/sdk/src/cli/cloud-deploy.ts index 644ee602e..53bc522b7 100644 --- a/packages/sdk/src/cli/cloud-deploy.ts +++ b/packages/sdk/src/cli/cloud-deploy.ts @@ -19,6 +19,7 @@ export interface CloudDeployArgs { /** Refuse a missing integration instead of offering to connect it. */ noConnect: boolean; json: boolean; + plugins: string[]; } /** @@ -39,6 +40,7 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | let noConnect = false; let json = false; const on: string[] = []; + const plugins: string[] = []; for (let i = 0; i < args.length; i++) { const arg = args[i]!; if (arg === '--json') { @@ -63,6 +65,13 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | i += 1; continue; } + if (arg === '--plugin') { + const next = args[i + 1]; + if (next === undefined || next.startsWith('-')) return undefined; + plugins.push(next); + i += 1; + continue; + } if (arg === '--repo' || arg === '--approver' || arg === '--name' || arg === '--on') { const next = args[i + 1]; if (next === undefined || next.startsWith('-')) return undefined; @@ -78,7 +87,7 @@ export function parseCloudDeployArgs(args: readonly string[]): CloudDeployArgs | value = arg; } if (value === undefined || repo === undefined || on.length === 0) return undefined; - return { command: 'cloud-deploy', value, repo, on, approver, name, agents, draft, noConnect, json }; + return { command: 'cloud-deploy', value, repo, on, approver, name, agents, draft, noConnect, json, plugins }; } function describeSource(source: FlowTriggerSource): string { @@ -109,6 +118,7 @@ export async function runCloudDeployCli(args: CloudDeployArgs, io: CliIo): Promi ...(args.name === undefined ? {} : { name: args.name }), ...(agents === undefined ? {} : { agents }), ...(connect === undefined ? {} : { connect }), + ...(args.plugins.length === 0 ? {} : { plugins: args.plugins }), }); if (args.json) { io.stdout(JSON.stringify({ ok: true, ...deployment })); diff --git a/packages/sdk/src/cloud-deploy.ts b/packages/sdk/src/cloud-deploy.ts index 6cf17520a..688b71b91 100644 --- a/packages/sdk/src/cloud-deploy.ts +++ b/packages/sdk/src/cloud-deploy.ts @@ -8,6 +8,7 @@ import { } from './cloud-http.js'; import { flowRequirements, type FlowRequirements } from './flow-requirements.js'; import { readProjectConfig } from './cli/check.js'; +import { assertNoUseDependencies, collectExtensionSubmissions } from './flow-extension-submit.js'; /** * Hosted listener deployment: the CLI form of the agentrelay.com onboarding's @@ -67,6 +68,11 @@ export interface DeployToCloudInput { connect?: ConnectPrompt; /** Skip the pre-submission integration check entirely (Cloud still checks on activation). */ checkConnections?: boolean; + /** + * Extra GitHub plugin refs resolved send-only (same path as `flows add`, + * without writing flows.json). Project-declared extensions are always sent. + */ + plugins?: readonly string[]; } export const FLOW_AGENT_HARNESSES = ['claude', 'codex'] as const; @@ -170,16 +176,8 @@ export async function deployToCloud( throw new CloudFlowError('unsupported_source', `${input.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}`); } - if (loaded.extensions.length > 0) { - // The deploy body carries one source file; a composed handler set has no - // wire form yet, so a deployment would silently lose the extensions. - throw new CloudFlowError('unsupported_source', - `Cloud deploy does not yet accept flow extensions (${loaded.extensions.map(e => e.name).join(', ')} composed by flows.json); deploy the base flow from a project without them.`); - } - if (loaded.graph.length !== 1) { - throw new CloudFlowError('unsupported_source', - 'Cloud deploys one self-contained .flow.ts source without use dependencies.'); - } + assertNoUseDependencies(loaded); + const extensions = await collectExtensionSubmissions(loaded, input.plugins ?? []); let projectCli: string | undefined; try { projectCli = readProjectConfig(dirname(resolve(input.path))).cli; @@ -243,6 +241,7 @@ export async function deployToCloud( inputs: { approver, agents }, repository: input.repository, sources, + ...(extensions.length === 0 ? {} : { extensions }), requirements: { integrations: requirements.integrations.map(i => i.provider), harnesses: requirements.harnesses, diff --git a/packages/sdk/src/cloud-run.ts b/packages/sdk/src/cloud-run.ts index cf60dc32c..343d44d51 100644 --- a/packages/sdk/src/cloud-run.ts +++ b/packages/sdk/src/cloud-run.ts @@ -9,6 +9,7 @@ import { CompileError, compileSpec, kernelToAuthoring, toKernelSpec } from './co import type { FlowSpec } from './spec.js'; import { snapshotJsonValue, type JsonValue } from './json-value.js'; import { loadAuthoredFlow, type SurfaceModuleAuthority } from './authored-flow-loader.js'; +import { assertNoUseDependencies, collectExtensionSubmissions, type FlowExtensionSubmission } from './flow-extension-submit.js'; import { CloudFlowError, cloudConnection, cloudFetch, cloudRequest, cloudRunId, isCloudRecord, type CloudConnectionOptions, @@ -61,6 +62,7 @@ export function cloudSubmissionBody(submission: CloudSubmission): Record { let spec: FlowSpec | undefined; - let authored: { source: string; authority: CloudAuthoredAuthority; name: string; schedules: ScheduleTriggerSource[] } | undefined; + let authored: { source: string; authority: CloudAuthoredAuthority; name: string; schedules: ScheduleTriggerSource[]; extensions: readonly FlowExtensionSubmission[] } | undefined; const inputPresent = Object.prototype.hasOwnProperty.call(options, 'input'); let authoredInput: JsonValue | undefined; try { @@ -122,18 +125,13 @@ export async function prepareCloudSubmission( `${flow.path} is not a loadable authored flow: ${error instanceof Error ? error.message : String(error)}. ` + 'Run `flows check` on it from the same directory.'); } - if (loaded.extensions.length > 0) { - throw new CloudFlowError('unsupported_source', - `Cloud authored submission does not yet accept flow extensions (${loaded.extensions.map(e => e.name).join(', ')} composed by flows.json).`); - } - if (loaded.graph.length !== 1) { - throw new CloudFlowError('unsupported_source', - 'Cloud authored submission currently accepts one self-contained .flow.ts source without use dependencies.'); - } + assertNoUseDependencies(loaded); + const extensions = await collectExtensionSubmissions(loaded); authored = { source, name: definition.name, schedules: definition.handlers.flatMap(h => h.trigger.kind === 'schedule' ? [h.trigger] : []), + extensions, authority: Object.freeze({ schemaVersion: 1, sourceSha256: createHash('sha256').update(bytes).digest('hex'), @@ -186,6 +184,7 @@ export async function prepareCloudSubmission( return { workflow: authored.source, fileType: 'ts', authoredAuthority: authored.authority, inputs: authoredInput, inputPresent: true, name: authored.name, schedules: authored.schedules, + ...(authored.extensions.length === 0 ? {} : { extensions: authored.extensions }), specHash: createHash('sha256').update(canonicalize({ authority: authored.authority, input: authoredInput })).digest('hex'), }; } diff --git a/packages/sdk/src/flow-extension-submit.ts b/packages/sdk/src/flow-extension-submit.ts new file mode 100644 index 000000000..997d0b80f --- /dev/null +++ b/packages/sdk/src/flow-extension-submit.ts @@ -0,0 +1,118 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { sha256 } from './bundle.js'; +import { CloudFlowError } from './cloud-http.js'; +import { extensionManifestOf } from './cli/add-extension.js'; +import { assertCompatible, runtimeVersions } from './flow-extension-compat.js'; +import type { LoadedAuthoredFlow } from './authored-flow-loader.js'; +import type { FlowExtensionManifest } from './flow-extension-manifest.js'; +import type { LoadedFlowExtension } from './flow-extension-loader.js'; +import { fetchGithubPlugin, MAX_PLUGIN_TOTAL_BYTES, resolveGithubSha, type FetchLike } from './plugin-github.js'; +import { PluginError } from './plugin-manifest.js'; +import { canonicalPluginRef, parsePluginSource } from './plugin-source.js'; +import { readStoredPluginFiles } from './plugin-store.js'; + +export const MAX_EXTENSIONS_BYTES = MAX_PLUGIN_TOTAL_BYTES; + +export interface FlowExtensionFileSubmission { + readonly path: string; + readonly sha256: string; + readonly bytes: number; + readonly encoding: 'utf8' | 'base64'; + readonly content: string; +} + +export interface FlowExtensionSubmission { + readonly name: string; + readonly version: string; + readonly ref: string; + readonly digest: string; + readonly manifestSha256: string; + readonly manifest: FlowExtensionManifest; + readonly files: readonly FlowExtensionFileSubmission[]; +} + +function encodeFile(path: string, data: Uint8Array): FlowExtensionFileSubmission { + const text = Buffer.from(data).toString('utf8'); + const utf8 = Buffer.from(text, 'utf8').equals(Buffer.from(data)); + return { + path, + sha256: sha256(data), + bytes: data.length, + encoding: utf8 ? 'utf8' : 'base64', + content: utf8 ? text : Buffer.from(data).toString('base64'), + }; +} + +function submissionSize(submission: FlowExtensionSubmission): number { + return submission.files.reduce((sum, file) => sum + file.bytes, 0); +} + +async function submissionFromStore(extension: LoadedFlowExtension): Promise { + const stored = await readStoredPluginFiles(extension.directory, extension.digest); + const files = stored.filter(file => file.path !== 'manifest.json').map(file => encodeFile(file.path, file.data)); + const manifestBytes = readFileSync(join(extension.directory, 'flows-plugin.json')); + return { + name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, + manifestSha256: sha256(manifestBytes), + manifest: extension.manifest, + files, + }; +} + +/** Resolve a GitHub plugin reference without writing the working tree. */ +export async function resolveExtensionSubmission( + input: string, + options: { fetch?: FetchLike; versions?: { sdk: string; surface: string } } = {}, +): Promise { + try { + const source = await resolveGithubSha(parsePluginSource(input), options.fetch); + const fetched = await fetchGithubPlugin(source, options.fetch); + const { manifest, manifestSha256 } = extensionManifestOf(fetched); + assertCompatible(manifest, options.versions ?? runtimeVersions()); + const files = fetched.files.map(file => encodeFile(file.path, file.data)); + return { + name: manifest.name, version: manifest.version, ref: canonicalPluginRef(source), + digest: fetched.digest, manifestSha256, manifest, files, + }; + } catch (error) { + if (error instanceof CloudFlowError) throw error; + if (error instanceof PluginError) throw new CloudFlowError('invalid_input', error.message); + throw new CloudFlowError('invalid_input', error instanceof Error ? error.message : String(error)); + } +} + +/** + * Installed extensions plus optional send-only `--plugin` refs. Caps the + * extensions field at 2 MB separately from the 256 KB source cap. + */ +export async function collectExtensionSubmissions( + loaded: LoadedAuthoredFlow, + extraRefs: readonly string[] = [], + options: { fetch?: FetchLike } = {}, +): Promise { + const submissions: FlowExtensionSubmission[] = []; + for (const extension of loaded.extensions) submissions.push(await submissionFromStore(extension)); + const names = new Set(submissions.map(s => s.name)); + for (const ref of extraRefs) { + const extra = await resolveExtensionSubmission(ref, options); + if (names.has(extra.name)) { + throw new CloudFlowError('invalid_input', `Plugin ${extra.name} is already in this project; omit --plugin or remove the installed copy.`); + } + names.add(extra.name); + submissions.push(extra); + } + const total = submissions.reduce((sum, item) => sum + submissionSize(item), 0); + if (total > MAX_EXTENSIONS_BYTES) { + throw new CloudFlowError('invalid_input', `Flow extensions exceed Cloud's ${MAX_EXTENSIONS_BYTES}-byte extensions cap.`); + } + return submissions; +} + +/** Graph nodes besides the root must be the composed extensions, not `use:` children. */ +export function assertNoUseDependencies(loaded: LoadedAuthoredFlow): void { + if (loaded.graph.length !== 1 + loaded.extensions.length) { + throw new CloudFlowError('unsupported_source', + 'Cloud deploys one self-contained .flow.ts source without use dependencies.'); + } +} diff --git a/packages/sdk/tests/cloud-deploy.test.ts b/packages/sdk/tests/cloud-deploy.test.ts index 696612f83..b9e8584af 100644 --- a/packages/sdk/tests/cloud-deploy.test.ts +++ b/packages/sdk/tests/cloud-deploy.test.ts @@ -118,6 +118,7 @@ describe('deployToCloud', () => { }); expect(body.handoffId).toMatch(/^flows-cli-[a-f0-9]{16}$/u); expect(body.source).toContain("flow<{ issue: { title: string }; approver: string }>('issue-triage'"); + expect(body.extensions).toBeUndefined(); expect(deployment).toMatchObject({ agentId: 'agent-1', status: 'listening', name: 'issue-triage', connected: [] }); expect(deployment.requirements.integrations.map(i => `${i.provider} (${i.detail})`)).toEqual(['github (--on github)', 'slack (--on slack)']); expect(deployment.sourceSha256).toMatch(/^[a-f0-9]{64}$/u); diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 6ae59889f..2c9bcea4f 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { collectExtensionSubmissions } from '../src/flow-extension-submit.js'; import { addExtensionPlugin } from '../src/cli/add-extension.js'; import { checkAuthoredTriggers } from '../src/cli/check-triggers.js'; import { runCli } from '../src/cli.js'; @@ -128,6 +129,12 @@ describe('composing flow extensions onto a base flow', () => { expect(await runCli(['check', p.flow], p.io)).toBe(0); expect(p.text()).toContain(`EXTENSION babysitter@0.1.0 ${REF} sha256:`); expect(p.text()).toContain('8 handler(s) composed after the base flow'); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + const submissions = await collectExtensionSubmissions(loaded); + expect(submissions).toHaveLength(1); + expect(submissions[0]).toMatchObject({ name: 'babysitter', ref: REF }); + expect(submissions[0]!.files.some(f => f.path === 'babysitter.flow.ts' && f.encoding === 'utf8')).toBe(true); + expect(submissions[0]!.files.reduce((n, f) => n + f.bytes, 0)).toBeGreaterThan(0); }); }); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index ff1297304..ef8b02b6f 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -79,7 +79,7 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa verb: 'deploy', argv: ['deploy', 'review.flow.ts', '--repo', 'owner/name', '--on', 'github:label=review', '--approver', 'someone', '--name', 'review-listener', '--agents', 'claude,codex', '--draft', - '--no-connect', '--json'], + '--plugin', 'github:o/r@main#path', '--no-connect', '--json'], variant: 'cloud-deploy', }, { verb: 'deployments', argv: ['deployments', '--json'], variant: 'deployments' }, From 34bbbd60e374c2de2447ba912713f7aefc2722ba Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 16:20:15 -0700 Subject: [PATCH 09/17] feat(catalog): list babysitter as a fail-closed schema-2 plugin (D1) catalog/plugins.json v1 records babysitter at the merge-gate hook sha with a digest of the plugin directory and an explicit note that ready_for_review/labeled/unlabeled stay plugin_event_unroutable until the relayfile adapter catalog grows. Co-Authored-By: Claude Opus 5 (1M context) --- catalog/plugins.json | 15 +++++++++ packages/sdk/tests/catalog-plugins.test.ts | 37 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 catalog/plugins.json create mode 100644 packages/sdk/tests/catalog-plugins.test.ts diff --git a/catalog/plugins.json b/catalog/plugins.json new file mode 100644 index 000000000..d7323b91e --- /dev/null +++ b/catalog/plugins.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "plugins": [ + { + "name": "babysitter", + "description": "Live-state PR babysitter: parallel review lenses, deterministic reconciliation, exact-head merge gate. Fail-closed: GitHub pull_request.ready_for_review, labeled, and unlabeled are not in the surface registry, so a manifest that declares them is refused plugin_event_unroutable until the relayfile adapter catalog grows.", + "source": { "owner": "AgentWorkforce", "repo": "flows", "path": "examples/babysitter" }, + "ref": "05c3dff138883322e80cb793b1f5a097ad510572", + "digest": "ae6af3335eb6d4e54559327acc1465419244b47911d8ff356850b61f6228d862", + "compat": { "surface": "^2.0.22", "sdk": "^2.0.22", "base": ["software-factory"] }, + "tier": "community", + "base": ["software-factory"] + } + ] +} diff --git a/packages/sdk/tests/catalog-plugins.test.ts b/packages/sdk/tests/catalog-plugins.test.ts new file mode 100644 index 000000000..6ee33050a --- /dev/null +++ b/packages/sdk/tests/catalog-plugins.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const SHA = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const TIERS = new Set(['first-party', 'verified', 'community']); + +describe('catalog/plugins.json', () => { + const catalog = JSON.parse(readFileSync(resolve('../../catalog/plugins.json'), 'utf8')) as { + version: unknown; + plugins: Array>; + }; + + it('is version 1 with unique kebab-case plugin names', () => { + expect(catalog.version).toBe(1); + expect(Array.isArray(catalog.plugins)).toBe(true); + expect(catalog.plugins.length).toBeGreaterThan(0); + const names = catalog.plugins.map(p => p.name); + expect(names.every(n => typeof n === 'string' && NAME.test(n))).toBe(true); + expect(new Set(names).size).toBe(names.length); + }); + + it('records a fail-closed babysitter entry with a pinned sha and digest', () => { + const babysitter = catalog.plugins.find(p => p.name === 'babysitter'); + expect(babysitter).toMatchObject({ + source: { owner: 'AgentWorkforce', repo: 'flows', path: 'examples/babysitter' }, + tier: 'community', + base: ['software-factory'], + }); + expect(babysitter!.ref).toMatch(SHA); + expect(babysitter!.digest).toMatch(HEX64); + expect(String(babysitter!.description)).toContain('plugin_event_unroutable'); + expect(TIERS.has(String(babysitter!.tier))).toBe(true); + }); +}); From d89e4ec81744ae1a84602dfa845f001bb4ea9f21 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 16:54:07 -0700 Subject: [PATCH 10/17] fix(sdk): type fake-github Response bodies without DOM BodyInit bundle.test.ts is in tsconfig.tests.json and now imports fake-github, so CI typecheck:tests sees BodyInit. Node's test tsconfig has no DOM lib; string | Uint8Array is what the double actually sends. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/tests/fake-github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/tests/fake-github.ts b/packages/sdk/tests/fake-github.ts index 19dde3098..c7df1e70f 100644 --- a/packages/sdk/tests/fake-github.ts +++ b/packages/sdk/tests/fake-github.ts @@ -36,7 +36,7 @@ export function fakeGithub(repos: Record): FakeGithub { const fetch: FetchLike = async (url) => { calls.push(url); const u = new URL(url); - const respond = (status: number, body: BodyInit | null = null, type = 'text/plain'): Response => new Response(body, { status, headers: { 'content-type': type } }); + const respond = (status: number, body: string | Uint8Array | null = null, type = 'text/plain'): Response => new Response(body, { status, headers: { 'content-type': type } }); if (u.host === 'api.github.com') { let m = /^\/repos\/([^/]+)\/([^/]+)\/commits\/(.+)$/.exec(u.pathname); if (m) { From 1a75d20e3d16e531023528bf4c4c2d3ca61c3b59 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 19:53:41 -0700 Subject: [PATCH 11/17] fix(sdk): allow version/hooks headers and fail closed on review findings Executor and flows check no longer treat FlowHeader.version/hooks as unsupported_header (Software Garden was unrunnable). Hook replay restores the parent step watermark. Wallclock ceilings are compared. Extension preflight is probed before the body. Hosted requirements union plugin permissions. The extensions cap is the serialized JSON size. Co-Authored-By: Claude Opus 5 (1M context) --- .../software-factory/software-factory.flow.ts | 13 +++++++++ packages/sdk/src/authored-flow-executor.ts | 9 ++++-- packages/sdk/src/authored-hooks.ts | 8 +++++ packages/sdk/src/cli/check-typescript.ts | 2 +- packages/sdk/src/cloud-deploy.ts | 15 ++++++++-- packages/sdk/src/flow-extension-loader.ts | 29 +++++++++++++++++++ packages/sdk/src/flow-extension-submit.ts | 6 +--- packages/sdk/tests/authored-flow.test.ts | 6 ++++ packages/sdk/tests/authored-hooks.test.ts | 6 +++- .../sdk/tests/flow-extension-compose.test.ts | 1 + 10 files changed, 83 insertions(+), 12 deletions(-) diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index 90312e072..336704e96 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -47,6 +47,11 @@ export default flow("software-factory", { // Fresh work dir, excluded from git, no leftover verdicts. await f.run(`rm -rf ${WORK} && mkdir -p ${WORK} && { grep -qxF '${WORK}/' .git/info/exclude 2>/dev/null || echo '${WORK}/' >> .git/info/exclude; }`); + if (!await f.hook("pre-implement", { title, issue })) { + await f.run("echo 'Stopped: pre-implement hook refused this ticket.' >&2"); + return f.done("declined"); + } + await f.agent("implementer", { cli: "claude", task: `Implement this ticket in the current repository, on the current branch, with regression tests. Commit as you go.\n` + @@ -66,6 +71,14 @@ export default flow("software-factory", { await f.run(TEST, { timeout: "15m" }); + if (!await f.hook("post-review", { title })) { + await f.run(`{ cat ${WORK}/summary.md; printf '\\n\\n## post-review: blocked\\n\\n'; } > ${WORK}/pr-body.md`); + await f.run("git add -A && (git diff --cached --quiet || git commit -qm 'Software factory: implementation and review fixes')"); + await f.run("git push --set-upstream origin HEAD"); + await f.run(`gh pr create --draft --title ${shellWord(`[blocked] ${title}`)} --body-file ${WORK}/pr-body.md`); + return f.done("step_failed"); + } + // Passed means exactly one verdict, and it is the pass marker. const verdict = await f.run(`if [ -f ${WORK}/review.passed ] && [ ! -f ${WORK}/review.blocked ]; then echo PASSED; else echo BLOCKED; fi`); await f.run("git add -A && (git diff --cached --quiet || git commit -qm 'Software factory: implementation and review fixes')"); diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 9d961e540..537c52e15 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -45,7 +45,7 @@ import { import { AuthoredFlowLifecycle } from './authored-flow-lifecycle.js'; import { JournalClient } from './journal-client.js'; import { createHookEvaluator } from './authored-hooks.js'; -import type { LoadedFlowExtension } from './flow-extension-loader.js'; +import { probeFlowExtension, type LoadedFlowExtension } from './flow-extension-loader.js'; import type { CompletionReason as ProtocolCompletionReason, RunCompletionReason as ProtocolRunCompletionReason, @@ -175,7 +175,7 @@ export async function executeAuthoredFlow( ...(options.onWait !== undefined ? { onWait: options.onWait } : {}), }; const definition = getDefinition(handle); - const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget' && key !== 'memory'); + const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget' && key !== 'memory' && key !== 'version' && key !== 'hooks'); if (definition.header.tools && Object.keys(definition.header.tools).some(key => !['mcp', ...helperProviders.map(p => p.namespace)].includes(key))) headerFields.push('tools'); if (definition.header.tools?.relayfile !== undefined) headerFields.push('tools.relayfile'); const helperPreflight = checkSlackHelpers(definition); @@ -192,6 +192,9 @@ export async function executeAuthoredFlow( const checkedMcp = await checkMcpHeader(definition, flowPath); if (!checkedMcp.report.ok) throw new McpPreflightError(checkedMcp.report); + for (const extension of options.extensions ?? []) { + if (extension.manifest !== undefined) await probeFlowExtension(extension.manifest); + } const budget = new AuthoredBudget(definition.header.budget); if (definition.header.memory?.agent === true) { @@ -359,6 +362,8 @@ export async function executeAuthoredFlow( flowName: definition.name, declared: definition.header.hooks ?? [], extensions: options.extensions ?? [], + peekStep: () => nextStep, + restoreStep: (step) => { nextStep = step; }, }); const context: Ctx = { diff --git a/packages/sdk/src/authored-hooks.ts b/packages/sdk/src/authored-hooks.ts index d361cf988..15599b376 100644 --- a/packages/sdk/src/authored-hooks.ts +++ b/packages/sdk/src/authored-hooks.ts @@ -11,6 +11,8 @@ export interface HookRecord { plugin: string | null; verdict: 'pass' | 'fail' | 'noop'; because?: string; + /** Parent `nextStep` after this verdict, so resume does not reuse inner step ids. */ + afterStep?: number; } function recordKey(record: HookRecord): string { @@ -29,6 +31,8 @@ export function createHookEvaluator(options: { readonly flowName: string; readonly declared: readonly string[]; readonly extensions: readonly LoadedFlowExtension[]; + readonly peekStep?: () => number; + readonly restoreStep?: (step: number) => void; }): (id: string, name: string, input: unknown, context: Ctx) => Promise { let recorded: Promise> | undefined; @@ -78,6 +82,7 @@ export function createHookEvaluator(options: { const existing = await lookup(`${id}:noop`); const record = existing ?? { hook: name, step: id, plugin: null, verdict: 'noop' as const }; if (existing === undefined) await append(record); + else if (existing.afterStep !== undefined) options.restoreStep?.(existing.afterStep); return true; } for (const extension of impls) { @@ -96,8 +101,11 @@ export function createHookEvaluator(options: { hook: name, step: id, plugin: extension.name, verdict: verdict ? 'pass' : 'fail', ...(because === undefined ? {} : { because }), + ...(options.peekStep === undefined ? {} : { afterStep: options.peekStep() }), }; await append(record); + } else if (record.afterStep !== undefined) { + options.restoreStep?.(record.afterStep); } if (record.verdict === 'fail') return false; } diff --git a/packages/sdk/src/cli/check-typescript.ts b/packages/sdk/src/cli/check-typescript.ts index cf35cba3d..8707e163a 100644 --- a/packages/sdk/src/cli/check-typescript.ts +++ b/packages/sdk/src/cli/check-typescript.ts @@ -28,7 +28,7 @@ export async function checkMcpHeader( path: string, ): Promise { const empty = { servers: Object.freeze({}), inventory: Object.freeze({}) }; - const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use']); + const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use', 'version', 'hooks']); const header = definition.header ?? {}; const unsupported = Object.keys(header).filter(key => !KNOWN_HEADER_FIELDS.has(key)); if (header.tools?.relayfile !== undefined) unsupported.push('tools.relayfile'); diff --git a/packages/sdk/src/cloud-deploy.ts b/packages/sdk/src/cloud-deploy.ts index 688b71b91..1965ffc3c 100644 --- a/packages/sdk/src/cloud-deploy.ts +++ b/packages/sdk/src/cloud-deploy.ts @@ -243,9 +243,18 @@ export async function deployToCloud( sources, ...(extensions.length === 0 ? {} : { extensions }), requirements: { - integrations: requirements.integrations.map(i => i.provider), - harnesses: requirements.harnesses, - mcp: requirements.mcp, + integrations: [...new Set([ + ...requirements.integrations.map(i => i.provider), + ...extensions.flatMap(extension => extension.manifest.permissions.integrations), + ])], + harnesses: [...new Set([ + ...requirements.harnesses, + ...extensions.flatMap(extension => extension.manifest.permissions.harnesses), + ])], + mcp: [...new Set([ + ...requirements.mcp, + ...extensions.flatMap(extension => extension.manifest.permissions.mcp), + ])], }, }) }); if (!isCloudRecord(result) || typeof result.agentId !== 'string' || typeof result.status !== 'string') { diff --git a/packages/sdk/src/flow-extension-loader.ts b/packages/sdk/src/flow-extension-loader.ts index 9dc845be2..14be52141 100644 --- a/packages/sdk/src/flow-extension-loader.ts +++ b/packages/sdk/src/flow-extension-loader.ts @@ -61,6 +61,28 @@ export interface LoadFlowExtensionsOptions { } const EXTENSION_HEADER_FIELDS = new Set(['budget', 'tools']); +const WALLCLOCK_MS = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 } as const; + +function wallclockMs(value: string): number | undefined { + const match = /^(\d+)(ms|s|m|h|d)$/.exec(value); + if (!match) return undefined; + const unit = match[2] as keyof typeof WALLCLOCK_MS; + return Number(match[1]) * WALLCLOCK_MS[unit]; +} + +/** Credentials and servers declared on a flow-extension, probed before the base body starts. */ +export async function probeFlowExtension(manifest: FlowExtensionManifest, env: NodeJS.ProcessEnv = process.env): Promise { + for (const credential of manifest.preflight.credentials) { + if (!env[credential]?.trim()) throw new PluginError('plugin_credential_missing', `${manifest.name} requires ${credential}.`); + } + for (const server of manifest.preflight.servers) { + try { + const response = await fetch(server, { method: 'HEAD', signal: AbortSignal.timeout(5000) }); + await response.body?.cancel(); + if (!response.ok) throw new Error('unsuccessful response'); + } catch { throw new PluginError('plugin_server_unreachable', `${manifest.name} cannot reach ${server}.`); } + } +} function unsupported(name: string, what: string): never { throw new PluginError('plugin_unsupported', `${name}: ${what} is not composed by this release.`); @@ -112,6 +134,13 @@ async function loadOne( if (ceiling?.dollars !== undefined && typeof baseBudget === 'object' && baseBudget.dollars !== undefined && ceiling.dollars > baseBudget.dollars) { throw new PluginError('plugin_incompatible', `${manifest.name} declares a $${ceiling.dollars} budget ceiling above the base flow's $${baseBudget.dollars}.`); } + if (ceiling?.wallclock !== undefined && typeof baseBudget === 'object' && baseBudget.wallclock !== undefined) { + const pluginMs = wallclockMs(ceiling.wallclock); + const baseMs = wallclockMs(baseBudget.wallclock); + if (pluginMs !== undefined && baseMs !== undefined && pluginMs > baseMs) { + throw new PluginError('plugin_incompatible', `${manifest.name} declares a ${ceiling.wallclock} wallclock ceiling above the base flow's ${baseBudget.wallclock}.`); + } + } const entryPath = join(directory, manifest.entry); let imported: ImportedFlow; try { imported = await options.importFlow(entryPath); } diff --git a/packages/sdk/src/flow-extension-submit.ts b/packages/sdk/src/flow-extension-submit.ts index 997d0b80f..d58f5c8d3 100644 --- a/packages/sdk/src/flow-extension-submit.ts +++ b/packages/sdk/src/flow-extension-submit.ts @@ -44,10 +44,6 @@ function encodeFile(path: string, data: Uint8Array): FlowExtensionFileSubmission }; } -function submissionSize(submission: FlowExtensionSubmission): number { - return submission.files.reduce((sum, file) => sum + file.bytes, 0); -} - async function submissionFromStore(extension: LoadedFlowExtension): Promise { const stored = await readStoredPluginFiles(extension.directory, extension.digest); const files = stored.filter(file => file.path !== 'manifest.json').map(file => encodeFile(file.path, file.data)); @@ -102,7 +98,7 @@ export async function collectExtensionSubmissions( names.add(extra.name); submissions.push(extra); } - const total = submissions.reduce((sum, item) => sum + submissionSize(item), 0); + const total = Buffer.byteLength(JSON.stringify(submissions), 'utf8'); if (total > MAX_EXTENSIONS_BYTES) { throw new CloudFlowError('invalid_input', `Flow extensions exceed Cloud's ${MAX_EXTENSIONS_BYTES}-byte extensions cap.`); } diff --git a/packages/sdk/tests/authored-flow.test.ts b/packages/sdk/tests/authored-flow.test.ts index ebe1fb449..32da08917 100644 --- a/packages/sdk/tests/authored-flow.test.ts +++ b/packages/sdk/tests/authored-flow.test.ts @@ -140,6 +140,12 @@ describe('authored flow journal executor', () => { async (f) => f.done('success'), ), disconnectedJournal)).rejects.toMatchObject({ code: 'unsupported_header' }); + await expect(executeAuthoredFlow(flow( + 'versioned-hooks', + { version: '2.0.22', hooks: ['merge-gate'], budget: { dollars: 1 } }, + async (f) => f.done('success'), + ), disconnectedJournal)).rejects.not.toMatchObject({ code: 'unsupported_header' }); + // Predicate .gate(fn) is accepted (its VERDICT is journaled as a lowered // `.gate` run once the step completes), so with a disconnected // journal it refuses on the run.start path like any other step. What is diff --git a/packages/sdk/tests/authored-hooks.test.ts b/packages/sdk/tests/authored-hooks.test.ts index 7c46d4bf3..3787cdaef 100644 --- a/packages/sdk/tests/authored-hooks.test.ts +++ b/packages/sdk/tests/authored-hooks.test.ts @@ -61,16 +61,20 @@ describe('hook AND composition', () => { it('replays a recorded verdict and does not re-run the closure', async () => { const { journal, streams } = fakeJournal(); streams.set('hooks', [ - { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'fail', because: 'held' }, + { hook: 'merge-gate', step: 'hook-1', plugin: 'first', verdict: 'fail', because: 'held', afterStep: 9 }, ]); let calls = 0; + let nextStep = 2; const evaluate = createHookEvaluator({ journal, rootRunId: 'root-1', flowName: 'software-factory', declared: ['merge-gate'], extensions: [extension('first', async () => { calls += 1; return true; })], + peekStep: () => nextStep, + restoreStep: (step) => { nextStep = step; }, }); await expect(evaluate('hook-1', 'merge-gate', {}, ctx)).resolves.toBe(false); expect(calls).toBe(0); + expect(nextStep).toBe(9); expect(streams.get('hooks')).toHaveLength(1); }); diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 2c9bcea4f..1b35d9520 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -164,6 +164,7 @@ describe('composition fails closed', () => { ['a base flow it does not extend', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'other-flow', version: '*' }] } }), 'plugin_incompatible', 'not "software-factory"'], ['a base version range the unversioned base cannot satisfy', (m: Record) => ({ ...m, compat: { ...(m.compat as object), base: [{ name: 'software-factory', version: '^1.0.0' }] } }), 'plugin_incompatible', 'declares no version'], ['a budget ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { dollars: 11 } } }), 'plugin_incompatible', 'above the base flow'], + ['a wallclock ceiling above the base', (m: Record) => ({ ...m, permissions: { ...(m.permissions as object), budget: { wallclock: '2h' } } }), 'plugin_incompatible', 'wallclock ceiling'], ])('refuses %s', async (_, patch, code, message) => { const p = project(); await install(p, variant(patch)); From 9530e3c574d77ae4b9301a50c0fe8ac25b0271f5 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 19:59:43 -0700 Subject: [PATCH 12/17] fix(sdk): address remaining flows #528 review comments Preserve PluginError codes through flows check, include extensions in the hosted specHash, verify nested plugin stores on --verify, treat legacy npm lockfile.json as v1, JSON-dry-run plugin update, atomic flows.json+lock writes, SemVer prerelease compare, safe URL decoding, manifest-order hook inspection, dotted GitHub repo names, and hook cancellation/timeout bounds. Co-Authored-By: Claude Opus 5 (1M context) --- testdata/plugins/extension-babysitter/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testdata/plugins/extension-babysitter/README.md b/testdata/plugins/extension-babysitter/README.md index 101773d74..bccfa64d3 100644 --- a/testdata/plugins/extension-babysitter/README.md +++ b/testdata/plugins/extension-babysitter/README.md @@ -7,9 +7,9 @@ Software Garden, served to the SDK tests by a fake GitHub (see example: the entry carries Babysitter's handler surface — one `.on()` per declared subscription — over a body that only declines, so composition onto a base flow can be proven without the real review body. `extends.hooks` is empty -because hooks are not composed by this release (a manifest declaring one is -refused with `plugin_unsupported`); the `merge-gate` hook from the design is a -later slice. +because this fixture does not declare a hook; declared hooks are composed when +the base flow names them. The `merge-gate` hook from the design is not included +in this fixture. Babysitter's own subscription contract (branch `feat/babysitter-v2`) names eleven GitHub subscriptions. Three of them — `pull_request.ready_for_review`, From bff1be83c67a952d1782d12da191e80e39f546b0 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 19:59:53 -0700 Subject: [PATCH 13/17] fix(sdk): remaining #528 review comments (codes, verify, JSON plan) Preserve PluginError through flows check, include extensions in specHash, verify nested plugin stores and reject unsafe names, accept legacy npm lockfile.json, emit the update plan in --json without --yes, write flows.json and the lock via temp+rename, SemVer prerelease compare, safe URL decoding, manifest-order hook inspection, dotted repo names, and hook cancellation/timeout bounds. Co-Authored-By: Claude Opus 5 (1M context) --- .../software-factory/software-factory.flow.ts | 2 +- packages/sdk/src/authored-flow-executor.ts | 1 + packages/sdk/src/authored-hooks.ts | 19 ++++++++++++- packages/sdk/src/bundle-extensions.ts | 27 +++++++++++++------ packages/sdk/src/cli.ts | 1 + packages/sdk/src/cli/add-extension.ts | 10 +++---- packages/sdk/src/cli/check-helper-body.ts | 4 +++ packages/sdk/src/cli/check-triggers.ts | 4 +-- packages/sdk/src/cli/check-typescript.ts | 4 +++ packages/sdk/src/cli/plugin.ts | 27 ++++++++++--------- packages/sdk/src/cloud-run.ts | 5 +++- packages/sdk/src/plugin-lock.ts | 13 ++++++++- packages/sdk/src/plugin-source.ts | 8 ++++-- packages/sdk/src/semver-range.ts | 24 ++++++++++++++++- packages/sdk/tests/plugin-extension.test.ts | 16 ++++++++++- 15 files changed, 129 insertions(+), 36 deletions(-) diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index 336704e96..91f778c18 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -89,7 +89,7 @@ export default flow("software-factory", { if (verdict.trim() === "PASSED") { const origin = (await f.run("git remote get-url origin")).trim(); const headSha = (await f.run("git rev-parse HEAD")).trim(); - const matched = /github\.com[:/]([^/]+)\/([^/.]+)/.exec(origin); + const matched = /github\.com[:/]([^/]+)\/([^/]+?)(?:\.git)?$/.exec(origin); const allowed = await f.hook("merge-gate", { owner: matched?.[1] ?? "", repo: matched?.[2] ?? "", diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 537c52e15..f57744005 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -364,6 +364,7 @@ export async function executeAuthoredFlow( extensions: options.extensions ?? [], peekStep: () => nextStep, restoreStep: (step) => { nextStep = step; }, + ...(options.signal === undefined ? {} : { signal: options.signal }), }); const context: Ctx = { diff --git a/packages/sdk/src/authored-hooks.ts b/packages/sdk/src/authored-hooks.ts index 15599b376..5fca2e3e0 100644 --- a/packages/sdk/src/authored-hooks.ts +++ b/packages/sdk/src/authored-hooks.ts @@ -33,6 +33,7 @@ export function createHookEvaluator(options: { readonly extensions: readonly LoadedFlowExtension[]; readonly peekStep?: () => number; readonly restoreStep?: (step: number) => void; + readonly signal?: AbortSignal; }): (id: string, name: string, input: unknown, context: Ctx) => Promise { let recorded: Promise> | undefined; @@ -92,7 +93,7 @@ export function createHookEvaluator(options: { let verdict = false; let because: string | undefined; try { - verdict = await extension.hooks[name]!(context, input) === true; + verdict = await boundHook(extension.hooks[name]!(context, input), options.signal) === true; } catch (error) { verdict = false; because = error instanceof Error ? error.message : String(error); @@ -112,3 +113,19 @@ export function createHookEvaluator(options: { return true; }; } + +const HOOK_DEADLINE_MS = 15 * 60 * 1000; + +async function boundHook(run: Promise, signal?: AbortSignal): Promise { + const timeout = AbortSignal.timeout(HOOK_DEADLINE_MS); + const abort = signal === undefined ? timeout : AbortSignal.any([signal, timeout]); + if (abort.aborted) throw new Error('hook cancelled'); + return await new Promise((resolve, reject) => { + const onAbort = () => reject(new Error('hook cancelled')); + abort.addEventListener('abort', onAbort, { once: true }); + run.then( + value => { abort.removeEventListener('abort', onAbort); resolve(value); }, + error => { abort.removeEventListener('abort', onAbort); reject(error); }, + ); + }); +} diff --git a/packages/sdk/src/bundle-extensions.ts b/packages/sdk/src/bundle-extensions.ts index 9bf92e308..8dd36978c 100644 --- a/packages/sdk/src/bundle-extensions.ts +++ b/packages/sdk/src/bundle-extensions.ts @@ -1,11 +1,11 @@ import { readFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import type { BundleFile } from './bundle.js'; -import { sha256 } from './bundle.js'; +import { safePath, sha256 } from './bundle.js'; import { findPluginProject } from './plugin-loader.js'; import { parsePluginLock, readPluginLock, reconcileDeclaredExtensions, type PluginLock } from './plugin-lock.js'; import { PluginError } from './plugin-manifest.js'; -import { pluginStoreDirectory, readStoredPluginFiles } from './plugin-store.js'; +import { pluginStoreDirectory, readStoredPluginFiles, verifyStoredPlugin } from './plugin-store.js'; const EMPTY_LOCK: PluginLock = Object.freeze({ version: 2, plugins: Object.freeze([]) }); @@ -36,18 +36,23 @@ export async function verifyBundlePluginLock(bundle: string): Promise { let parsed: unknown; try { parsed = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); } catch { throw new Error('lockfile.json: not valid JSON'); } - if (isLegacyV1Lock(parsed)) return; + if (isLegacyV1Lock(parsed) || isLegacyNpmLock(parsed)) return; let lock: PluginLock; try { lock = parsePluginLock(parsed); } catch (error) { throw new Error(error instanceof PluginError ? error.message : 'lockfile.json: not a plugin lock'); } for (const entry of lock.plugins) { - let manifest: Buffer; - try { manifest = await readFile(join(bundle, 'plugins', entry.name, 'manifest.json')); } - catch { throw new Error(`lockfile.json: plugin ${entry.name} is missing plugins/${entry.name}/manifest.json`); } - if (sha256(manifest) !== entry.digest) { - throw new Error(`lockfile.json: plugin ${entry.name} digest does not match plugins/${entry.name}/manifest.json`); + if (!safePath(entry.name) || entry.name.includes('/')) { + throw new Error(`lockfile.json: plugin name ${entry.name} is not a safe path component`); + } + const directory = join(bundle, 'plugins', entry.name); + await verifyStoredPlugin(directory, entry.digest); + let pluginManifest: Buffer; + try { pluginManifest = await readFile(join(directory, 'flows-plugin.json')); } + catch { throw new Error(`lockfile.json: plugin ${entry.name} is missing plugins/${entry.name}/flows-plugin.json`); } + if (sha256(pluginManifest) !== entry.manifestSha256) { + throw new Error(`lockfile.json: plugin ${entry.name} flows-plugin.json does not match the lockfile manifest hash`); } } } @@ -57,3 +62,9 @@ function isLegacyV1Lock(value: unknown): boolean { && (value as { version?: unknown }).version === 1 && Array.isArray((value as { adapters?: unknown }).adapters); } + +function isLegacyNpmLock(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const lock = value as { lockfileVersion?: unknown; packages?: unknown }; + return (lock.lockfileVersion === 2 || lock.lockfileVersion === 3) && typeof lock.packages === 'object' && lock.packages !== null; +} diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 60569857f..202a69a4a 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -356,6 +356,7 @@ async function checkAuthoredFlowComposed(path: string): Promise<{ report: CheckR ...mcp.report, ...(triggers?.report.schedules === undefined ? {} : { schedules: triggers.report.schedules }), ...(triggers?.report.extensions === undefined ? {} : { extensions: triggers.report.extensions }), + ...(triggers?.report.hooks === undefined ? {} : { hooks: triggers.report.hooks }), // The authored definition sees helper flags, body use and `cli:` // declarations; the compiled view underneath knows only its steps. ...(triggers?.report.requirements === undefined ? {} : { requirements: triggers.report.requirements }), diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts index 8596bee39..4f8196255 100644 --- a/packages/sdk/src/cli/add-extension.ts +++ b/packages/sdk/src/cli/add-extension.ts @@ -1,11 +1,11 @@ -import { readFileSync, writeFileSync } from 'node:fs'; +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { sha256 } from '../bundle.js'; import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha, type FetchLike, type FetchedPlugin } from '../plugin-github.js'; -import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, writePluginLock } from '../plugin-lock.js'; +import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, writeFlowsAndLock } from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; @@ -118,14 +118,12 @@ export async function addExtensionPlugin(input: string, io: CliIo, options: AddE source: { host: 'github', owner: source.owner, repo: source.repo, sha: source.sha, path: source.path }, digest, manifestSha256, resolvedAt: (options.now ?? (() => new Date()))().toISOString(), }); - config.plugins = plugins; - writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); - writePluginLock(root, next); + writeFlowsAndLock(root, configPath, config, plugins, next); io.stdout(`Added ${manifest.name}@${manifest.version} (flow-extension) from ${ref}`); io.stdout(` digest sha256:${digest}`); io.stdout(` materialized at ${directory}`); for (const line of describeExtension(manifest)) io.stdout(line); - io.stdout(` recorded in flows.json and ${PLUGIN_LOCK_FILE}; runtime composition is not yet supported (plugin_unsupported at run time)`); + io.stdout(` recorded in flows.json and ${PLUGIN_LOCK_FILE}`); return 0; } catch (error) { const refusal = error instanceof PluginError ? error : new PluginError('plugin_manifest_invalid', (error as Error).message); diff --git a/packages/sdk/src/cli/check-helper-body.ts b/packages/sdk/src/cli/check-helper-body.ts index febc5dcde..5a156400c 100644 --- a/packages/sdk/src/cli/check-helper-body.ts +++ b/packages/sdk/src/cli/check-helper-body.ts @@ -1,4 +1,5 @@ import { loadAuthoredFlow } from '../authored-flow-loader.js'; +import { PluginError } from '../plugin-manifest.js'; import { checkSlackHelpers } from '../slack-preflight.js'; import { inputFailureReport, type CheckExecution } from './check.js'; @@ -8,6 +9,9 @@ export async function checkHelperBody(path: string): Promise { const { handle, getDefinition } = await loadAuthoredFlow(path); return { report: { ...checkSlackHelpers(getDefinition(handle)), path } }; } catch (error) { + if (error instanceof PluginError) { + return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; + } return { report: inputFailureReport({ kind: 'invalid_spec', message: error instanceof Error ? error.message : 'Could not import authored flow.' }, path) }; } diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index 4222b557e..b87de17f6 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -45,11 +45,11 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const extensions = (loaded.extensions ?? []).map(extension => ({ name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, handlers: extension.handlers.length, - hooks: Object.keys(extension.hooks ?? {}), + hooks: extension.manifest.extends.hooks, })); const declaredHooks = definition.header.hooks ?? []; const implementations = (loaded.extensions ?? []).flatMap(extension => - Object.keys(extension.hooks ?? {}).map(hook => ({ hook, plugin: extension.name }))); + extension.manifest.extends.hooks.map(hook => ({ hook, plugin: extension.name }))); const hooks = declaredHooks.length > 0 || implementations.length > 0 ? { declared: declaredHooks, implementations } : undefined; return { diff --git a/packages/sdk/src/cli/check-typescript.ts b/packages/sdk/src/cli/check-typescript.ts index 8707e163a..33b1de54a 100644 --- a/packages/sdk/src/cli/check-typescript.ts +++ b/packages/sdk/src/cli/check-typescript.ts @@ -2,6 +2,7 @@ import type { LoadedPlugin } from '../plugin-loader.js'; import { dirname, resolve } from 'node:path'; import type { AuthoredFlowDefinition } from '../authored-flow.js'; import { loadAuthoredFlow } from '../authored-flow-loader.js'; +import { PluginError } from '../plugin-manifest.js'; import { preflight } from '../preflight.js'; import { SPEC_SCHEMA_VERSION, type McpServerConfig } from '../spec.js'; import { inputFailureReport, readProjectConfig, type CheckReport } from './check.js'; @@ -18,6 +19,9 @@ export async function checkTypeScriptFlow(path: string): Promise<{ report: Check const { handle, getDefinition } = await loadAuthoredFlow(path); return await checkMcpHeader(getDefinition(handle), path); } catch (error) { + if (error instanceof PluginError) { + return { report: inputFailureReport({ kind: error.code, message: error.message }, path) }; + } return { report: inputFailureReport({ kind: 'invalid_spec', message: (error as Error).message }, path) }; } } diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts index b4ec4f17d..085574b4c 100644 --- a/packages/sdk/src/cli/plugin.ts +++ b/packages/sdk/src/cli/plugin.ts @@ -1,4 +1,4 @@ -import { readFileSync, writeFileSync } from 'node:fs'; +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { diffExtension, extensionManifestOf } from './add-extension.js'; @@ -7,7 +7,7 @@ import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../fl import { fetchGithubPlugin, resolveGithubSha, type FetchLike } from '../plugin-github.js'; import { PLUGIN_LOCK_FILE, lockForDeclared, lockWithPlugin, lockedPlugins, readPluginLock, reconcileDeclaredExtensions, - writePluginLock, type PluginLock, type PluginLockEntry, + writeFlowsAndLock, type PluginLock, type PluginLockEntry, } from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; @@ -134,11 +134,6 @@ function readFlowsConfig(root: string): { path: string; config: Record, plugins: readonly string[]): void { - config.plugins = [...plugins]; - writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`); -} - function storedManifest(directory: string): FlowExtensionManifest { let raw: string; try { raw = readFileSync(join(directory, 'flows-plugin.json'), 'utf8'); } @@ -165,8 +160,7 @@ async function removePlugin(root: string, parsed: Extract ref !== match.ref); const nextLock = lockForDeclared(readPluginLock(root), nextDeclared); - writeFlowsPlugins(path, config, nextDeclared); - writePluginLock(root, nextLock); + writeFlowsAndLock(root, path, config, nextDeclared, nextLock); await dropUnreferencedStore(root, match.entry.name, match.entry.digest, nextLock); if (parsed.json) { io.stdout(JSON.stringify({ ok: true, removed: { name: match.entry.name, ref: match.ref, digest: match.entry.digest } })); @@ -256,7 +250,17 @@ async function updatePlugins( if (parsed.json) io.stdout(JSON.stringify({ ok: true, plugins: summary })); return 0; } - if (!parsed.yes) throw new PluginError('plugin_manifest_invalid', 'Re-run with --yes to apply this update.'); + if (!parsed.yes) { + if (parsed.json) { + io.stdout(JSON.stringify({ + ok: false, applied: false, code: 'plugin_manifest_invalid', + message: 'Re-run with --yes to apply this update.', plugins: summary, + })); + io.stderr('REFUSED [plugin_manifest_invalid] Re-run with --yes to apply this update.'); + return 2; + } + throw new PluginError('plugin_manifest_invalid', 'Re-run with --yes to apply this update.'); + } let declared = readFlowsConfig(root).plugins; let lock = readPluginLock(root); @@ -271,8 +275,7 @@ async function updatePlugins( resolvedAt: (options.now ?? (() => new Date()))().toISOString(), }); const { path, config } = readFlowsConfig(root); - writeFlowsPlugins(path, config, declared); - writePluginLock(root, lock); + writeFlowsAndLock(root, path, config, declared, lock); await dropUnreferencedStore(root, plan.current.entry.name, plan.current.entry.digest, lock); applied.push({ name: plan.manifest.name, ref: plan.ref, digest }); } diff --git a/packages/sdk/src/cloud-run.ts b/packages/sdk/src/cloud-run.ts index 343d44d51..aee830eca 100644 --- a/packages/sdk/src/cloud-run.ts +++ b/packages/sdk/src/cloud-run.ts @@ -185,7 +185,10 @@ export async function prepareCloudSubmission( workflow: authored.source, fileType: 'ts', authoredAuthority: authored.authority, inputs: authoredInput, inputPresent: true, name: authored.name, schedules: authored.schedules, ...(authored.extensions.length === 0 ? {} : { extensions: authored.extensions }), - specHash: createHash('sha256').update(canonicalize({ authority: authored.authority, input: authoredInput })).digest('hex'), + specHash: createHash('sha256').update(canonicalize({ + authority: authored.authority, input: authoredInput, + extensions: authored.extensions.map(extension => ({ name: extension.name, digest: extension.digest, ref: extension.ref })), + })).digest('hex'), }; } diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index 000aec90e..63cf67cd7 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { PluginError } from './plugin-manifest.js'; import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; @@ -75,6 +75,17 @@ export function writePluginLock(root: string, lock: PluginLock): void { writeFileSync(join(root, PLUGIN_LOCK_FILE), `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); } +/** Write declaration and lock temps, then rename both so a crash cannot leave one updated. */ +export function writeFlowsAndLock(root: string, configPath: string, config: Record, plugins: readonly string[], lock: PluginLock): void { + const jsonTmp = `${configPath}.tmp`; + const lockPath = join(root, PLUGIN_LOCK_FILE); + const lockTmp = `${lockPath}.tmp`; + writeFileSync(jsonTmp, `${JSON.stringify({ ...config, plugins: [...plugins] }, null, 2)}\n`); + writeFileSync(lockTmp, `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); + renameSync(lockTmp, lockPath); + renameSync(jsonTmp, configPath); +} + /** * Rebuild the entry list in `flows.json.plugins` order: `order` is derived from * the declaration list, never stored independently, so the two cannot disagree. diff --git a/packages/sdk/src/plugin-source.ts b/packages/sdk/src/plugin-source.ts index a9923dbe6..7eedcea56 100644 --- a/packages/sdk/src/plugin-source.ts +++ b/packages/sdk/src/plugin-source.ts @@ -59,8 +59,12 @@ export function parsePluginSource(input: string): PluginSourceInput { if (url.username || url.password || url.search || url.hash) return invalid('Plugin URL must not carry credentials, a query, or a fragment.'); const m = /^\/([^/]+)\/([^/]+)\/(?:tree|blob)\/([^/]+)(?:\/(.*))?$/.exec(url.pathname); if (!m) return invalid('Expected https://github.com///tree//.'); - [, owner, repo, ref] = m.map(part => part === undefined ? part : decodeURIComponent(part)); - path = m[4] === undefined ? '' : decodeURIComponent(m[4]); + try { + [, owner, repo, ref] = m.map(part => part === undefined ? part : decodeURIComponent(part)); + path = m[4] === undefined ? '' : decodeURIComponent(m[4]); + } catch { + return invalid('Plugin URL contains invalid percent-encoding.'); + } } else return invalid('Expected a github: reference or a https://github.com/ URL.'); if (owner === undefined || !OWNER.test(owner)) return invalid('Invalid GitHub owner.'); if (repo === undefined || !REPO.test(repo) || repo === '.' || repo === '..') return invalid('Invalid GitHub repository name.'); diff --git a/packages/sdk/src/semver-range.ts b/packages/sdk/src/semver-range.ts index b09532499..3a2cfc8ed 100644 --- a/packages/sdk/src/semver-range.ts +++ b/packages/sdk/src/semver-range.ts @@ -29,7 +29,29 @@ function compare(a: Parsed, b: Parsed): number { if (a.pre === b.pre) return 0; if (a.pre === undefined) return 1; if (b.pre === undefined) return -1; - return a.pre < b.pre ? -1 : 1; + return comparePrerelease(a.pre, b.pre); +} + +function comparePrerelease(a: string, b: string): number { + const as = a.split('.'); + const bs = b.split('.'); + const n = Math.max(as.length, bs.length); + for (let i = 0; i < n; i++) { + const left = as[i]; + const right = bs[i]; + if (left === undefined) return -1; + if (right === undefined) return 1; + const leftNum = /^\d+$/.test(left) ? Number(left) : undefined; + const rightNum = /^\d+$/.test(right) ? Number(right) : undefined; + if (leftNum !== undefined && rightNum !== undefined) { + if (leftNum !== rightNum) return leftNum - rightNum; + continue; + } + if (leftNum !== undefined) return -1; + if (rightNum !== undefined) return 1; + if (left !== right) return left < right ? -1 : 1; + } + return 0; } /** Whether `version` satisfies `range`; false for malformed input rather than a throw. */ diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts index 481713f01..e112abfc4 100644 --- a/packages/sdk/tests/plugin-extension.test.ts +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -54,6 +54,7 @@ describe('plugin source references', () => { 'github:o/r@main#../etc', 'github:o/r@main#a/../b', 'github:o/r@main#/abs', 'github:o/r@../x', 'github:o/r@main#a\\b', 'https://github.com/o/r/tree/main/x?token=1', 'https://user:pw@github.com/o/r/tree/main/x', 'https://gitlab.com/o/r/tree/main/x', 'github:o/r', 'github:o/r@main#examples/flows-plugin.json', 'github:-bad/r@main', 'github:o/r@main.lock', + 'https://github.com/o/r/tree/main/%E0%A4%A', ])('refuses %s', input => { expect(() => parsePluginSource(input)).toThrow(expect.objectContaining({ code: 'plugin_source_invalid' })); }); @@ -71,6 +72,7 @@ describe('semver ranges', () => { ['2.0.30', '~2.0.22', true], ['2.1.0', '~2.0.22', false], ['5.0.0', '>=2.0.0', true], ['2.5.0', '>=2.0.0 <2.5.0', false], ['2.0.22', '2.0.22', true], ['2.0.23', '2.0.22', false], ['0.0.9', '*', true], ['0.1.5', '^0.1.0', true], ['0.2.0', '^0.1.0', false], ['2.0.22', 'latest', false], ['x', '*', false], + ['1.0.0-alpha.10', '>=1.0.0-alpha.2', true], ['1.0.0-alpha.2', '>=1.0.0-alpha.10', false], ])('%s satisfies %s → %s', (version, range, ok) => { expect(satisfiesRange(version, range)).toBe(ok); }); }); @@ -92,7 +94,7 @@ describe('flows add ', () => { expect(p.text()).toContain(`Added babysitter@0.1.0 (flow-extension) from ${REF}`); expect(p.text()).toContain('events: github pull_request[opened,synchronize,reopened,closed]; github pull_request_review[submitted,dismissed]; github check_run[completed]; github issue_comment[created]'); expect(p.text()).toContain('writes (declared, unenforced): github:pull_request:comment'); - expect(p.text()).toContain('runtime composition is not yet supported'); + expect(p.text()).toContain('recorded in flows.json and flows.lock.json'); expect(gh.calls.some(url => url.includes('/commits/feat%2Fbabysitter-v2'))).toBe(true); // A tag naming the same commit is a no-op re-add: no duplicate declaration, same lock entry. expect(await addPlugin('https://github.com/AgentWorkforce/flows/tree/v0.1.0/examples/babysitter', p.io, { cwd: p.cwd, extension: { fetch: gh.fetch, now, versions } })).toBe(0); @@ -354,6 +356,18 @@ describe('flows plugin remove / update', () => { })).toBe(0); expect(JSON.parse(p.text())).toMatchObject({ ok: true, plugins: [{ name: 'babysitter', changed: false, digest }] }); }); + it('emits the permissions diff in JSON without --yes and does not rewrite the lock', async () => { + const { p, digest } = await installed(); + const updated = github(withManifest(m => ({ ...m, version: '0.2.0' }))); + updated.repos['AgentWorkforce/flows']!.commits[SHA_B] = updated.repos['AgentWorkforce/flows']!.commits[SHA_A]!; + const to = `github:AgentWorkforce/flows@${SHA_B}#examples/babysitter`; + p.messages.length = 0; + expect(await runPluginCommand({ command: 'plugin', sub: 'update', json: true, yes: false, name: 'babysitter', to }, p.io, { + cwd: p.cwd, fetch: updated.fetch, now, versions, + })).toBe(2); + expect(JSON.parse(p.messages.find(line => line.startsWith('{'))!)).toMatchObject({ ok: false, applied: false, code: 'plugin_manifest_invalid', plugins: [{ name: 'babysitter', changed: true }] }); + expect(readPluginLock(p.cwd).plugins[0]!.digest).toBe(digest); + }); it('parses the new subcommands and refuses a malformed invocation', () => { expect(parsePluginArgs(['remove', 'babysitter'])).toEqual({ command: 'plugin', sub: 'remove', json: false, name: 'babysitter' }); expect(parsePluginArgs(['update', '--yes'])).toEqual({ command: 'plugin', sub: 'update', json: false, yes: true, name: undefined, to: undefined }); From 7f04586a2d3e8af78965a81b5506276cb0f2b31d Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 20:01:22 -0700 Subject: [PATCH 14/17] fix(sdk): tolerate partial authored definitions when reading hooks direct-run tests (and any loader double) supply getDefinition() => ({}). definition.header.hooks threw and flows check reported invalid_spec instead of the authored failure. Optional-chain header and manifest. Co-Authored-By: Claude Opus 5 (1M context) --- packages/sdk/src/authored-flow-executor.ts | 2 +- packages/sdk/src/cli/check-triggers.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index f57744005..bbd66d05e 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -360,7 +360,7 @@ export async function executeAuthoredFlow( journal, ...(options.rootRunId === undefined ? {} : { rootRunId: options.rootRunId }), flowName: definition.name, - declared: definition.header.hooks ?? [], + declared: definition.header?.hooks ?? [], extensions: options.extensions ?? [], peekStep: () => nextStep, restoreStep: (step) => { nextStep = step; }, diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index b87de17f6..324ac7fa0 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -45,11 +45,11 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const extensions = (loaded.extensions ?? []).map(extension => ({ name: extension.name, version: extension.version, ref: extension.ref, digest: extension.digest, handlers: extension.handlers.length, - hooks: extension.manifest.extends.hooks, + hooks: extension.manifest?.extends.hooks ?? [], })); - const declaredHooks = definition.header.hooks ?? []; + const declaredHooks = definition.header?.hooks ?? []; const implementations = (loaded.extensions ?? []).flatMap(extension => - extension.manifest.extends.hooks.map(hook => ({ hook, plugin: extension.name }))); + (extension.manifest?.extends.hooks ?? []).map(hook => ({ hook, plugin: extension.name }))); const hooks = declaredHooks.length > 0 || implementations.length > 0 ? { declared: declaredHooks, implementations } : undefined; return { From 3e91344449e90681370f1a57460718b9f624184d Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 21 Sep 2026 07:59:36 +0200 Subject: [PATCH 15/17] fix(sdk): close flow-extension preflight gaps Session-Id: 01a0c26e-544c-7f11-b213-5cd7a2efabfa --- packages/sdk/src/authored-hooks.ts | 23 +++--- packages/sdk/src/cli/check-triggers.ts | 11 ++- packages/sdk/src/cloud-deploy.ts | 28 +++---- packages/sdk/src/cloud-run.ts | 6 +- packages/sdk/src/flow-extension-submit.ts | 8 +- packages/sdk/src/flow-requirements.ts | 47 +++++++++++- packages/sdk/tests/authored-hooks.test.ts | 15 ++++ packages/sdk/tests/bundle.test.ts | 3 +- .../sdk/tests/flow-extension-compose.test.ts | 73 ++++++++++++++++++- packages/sdk/tests/flow-requirements.test.ts | 22 +++++- 10 files changed, 200 insertions(+), 36 deletions(-) diff --git a/packages/sdk/src/authored-hooks.ts b/packages/sdk/src/authored-hooks.ts index 5fca2e3e0..fc383a501 100644 --- a/packages/sdk/src/authored-hooks.ts +++ b/packages/sdk/src/authored-hooks.ts @@ -95,6 +95,11 @@ export function createHookEvaluator(options: { try { verdict = await boundHook(extension.hooks[name]!(context, input), options.signal) === true; } catch (error) { + // Cancellation is control-plane state, not a plugin verdict. The + // shared execution signal also stops tracked f.run/f.agent work; + // rethrow so a later resume may run the hook again instead of + // replaying a permanent decline caused by an interrupted attempt. + if (options.signal?.aborted) throw error; verdict = false; because = error instanceof Error ? error.message : String(error); } @@ -114,18 +119,18 @@ export function createHookEvaluator(options: { }; } -const HOOK_DEADLINE_MS = 15 * 60 * 1000; - async function boundHook(run: Promise, signal?: AbortSignal): Promise { - const timeout = AbortSignal.timeout(HOOK_DEADLINE_MS); - const abort = signal === undefined ? timeout : AbortSignal.any([signal, timeout]); - if (abort.aborted) throw new Error('hook cancelled'); + // The parent run's wallclock budget owns the deadline. A second timer here + // would not be the signal captured by the hook's Ctx operations, allowing + // those operations to continue after this wrapper returned. + if (signal === undefined) return await run; + if (signal.aborted) throw signal.reason ?? new Error('hook cancelled'); return await new Promise((resolve, reject) => { - const onAbort = () => reject(new Error('hook cancelled')); - abort.addEventListener('abort', onAbort, { once: true }); + const onAbort = () => reject(signal.reason ?? new Error('hook cancelled')); + signal.addEventListener('abort', onAbort, { once: true }); run.then( - value => { abort.removeEventListener('abort', onAbort); resolve(value); }, - error => { abort.removeEventListener('abort', onAbort); reject(error); }, + value => { signal.removeEventListener('abort', onAbort); resolve(value); }, + error => { signal.removeEventListener('abort', onAbort); reject(error); }, ); }); } diff --git a/packages/sdk/src/cli/check-triggers.ts b/packages/sdk/src/cli/check-triggers.ts index 324ac7fa0..d938fb7f0 100644 --- a/packages/sdk/src/cli/check-triggers.ts +++ b/packages/sdk/src/cli/check-triggers.ts @@ -1,10 +1,11 @@ import { dirname, resolve } from 'node:path'; import { loadAuthoredFlow, type LoadedAuthoredFlow } from '../authored-flow-loader.js'; +import { probeFlowExtension } from '../flow-extension-loader.js'; import { preflightWebhookTriggers } from '../preflight.js'; import { preflightProviderTriggers } from '../provider-trigger-contract.js'; import { scheduleLowering } from '../schedule-trigger.js'; import { checkSlackHelpers } from '../slack-preflight.js'; -import { flowRequirements } from '../flow-requirements.js'; +import { flowRequirements, mergeFlowExtensionRequirements } from '../flow-requirements.js'; import { PluginError } from '../plugin-manifest.js'; import { inputFailureReport, readProjectConfig, type CheckReport } from './check.js'; @@ -23,6 +24,7 @@ export async function checkAuthoredTriggers(path: string): Promise<{ const loaded = await loadAuthoredFlow(path); const definition = loaded.getDefinition(loaded.handle); const config = readProjectConfig(dirname(resolve(path))); + for (const extension of loaded.extensions ?? []) await probeFlowExtension(extension.manifest); const triggers = (definition.handlers ?? []).map(handler => handler.trigger); const triggerDiagnostics = preflightWebhookTriggers(triggers, config.executors); // Registration answers "may this inbox run here"; the provider contract @@ -61,7 +63,12 @@ export async function checkAuthoredTriggers(path: string): Promise<{ ...(schedules.length === 0 ? {} : { schedules }), ...(extensions.length === 0 ? {} : { extensions }), ...(hooks === undefined ? {} : { hooks }), - requirements: flowRequirements(definition, { projectCli: config.cli }), + requirements: mergeFlowExtensionRequirements( + flowRequirements(definition, { projectCli: config.cli }), + (loaded.extensions ?? []).map(extension => ({ + name: extension.name, permissions: extension.manifest.permissions, + })), + ), ...(config.path === undefined ? {} : { projectConfigPath: config.path }), }, }; diff --git a/packages/sdk/src/cloud-deploy.ts b/packages/sdk/src/cloud-deploy.ts index 1965ffc3c..a83d3f94e 100644 --- a/packages/sdk/src/cloud-deploy.ts +++ b/packages/sdk/src/cloud-deploy.ts @@ -6,7 +6,9 @@ import { ensureIntegrationsConnected, type ConnectPrompt } from './cloud-connect import { CloudFlowError, cloudFetch, cloudRequest, isCloudRecord, type CloudConnectionOptions, } from './cloud-http.js'; -import { flowRequirements, type FlowRequirements } from './flow-requirements.js'; +import { + flowRequirements, mergeFlowExtensionRequirements, type FlowRequirements, +} from './flow-requirements.js'; import { readProjectConfig } from './cli/check.js'; import { assertNoUseDependencies, collectExtensionSubmissions } from './flow-extension-submit.js'; @@ -208,9 +210,12 @@ export async function deployToCloud( } // Every launched run lands in the deployment's repository, so GitHub is // required even when no GitHub source wakes it. - const requirements = flowRequirements(definition, { - sources, repository: input.repository, ...(projectCli === undefined ? {} : { projectCli }), - }); + const requirements = mergeFlowExtensionRequirements( + flowRequirements(definition, { + sources, repository: input.repository, ...(projectCli === undefined ? {} : { projectCli }), + }), + extensions.map(extension => ({ name: extension.name, permissions: extension.manifest.permissions })), + ); // The declared harnesses become `inputs.agents`; one Cloud cannot run is // refused here rather than silently replaced by Claude, which activation // would then check while the deployed runs still call the declared CLI. @@ -243,18 +248,9 @@ export async function deployToCloud( sources, ...(extensions.length === 0 ? {} : { extensions }), requirements: { - integrations: [...new Set([ - ...requirements.integrations.map(i => i.provider), - ...extensions.flatMap(extension => extension.manifest.permissions.integrations), - ])], - harnesses: [...new Set([ - ...requirements.harnesses, - ...extensions.flatMap(extension => extension.manifest.permissions.harnesses), - ])], - mcp: [...new Set([ - ...requirements.mcp, - ...extensions.flatMap(extension => extension.manifest.permissions.mcp), - ])], + integrations: requirements.integrations.map(i => i.provider), + harnesses: requirements.harnesses, + mcp: requirements.mcp, }, }) }); if (!isCloudRecord(result) || typeof result.agentId !== 'string' || typeof result.status !== 'string') { diff --git a/packages/sdk/src/cloud-run.ts b/packages/sdk/src/cloud-run.ts index aee830eca..fb92081e9 100644 --- a/packages/sdk/src/cloud-run.ts +++ b/packages/sdk/src/cloud-run.ts @@ -187,7 +187,11 @@ export async function prepareCloudSubmission( ...(authored.extensions.length === 0 ? {} : { extensions: authored.extensions }), specHash: createHash('sha256').update(canonicalize({ authority: authored.authority, input: authoredInput, - extensions: authored.extensions.map(extension => ({ name: extension.name, digest: extension.digest, ref: extension.ref })), + ...(authored.extensions.length === 0 ? {} : { + extensions: authored.extensions.map(extension => ({ + name: extension.name, digest: extension.digest, ref: extension.ref, + })), + }), })).digest('hex'), }; } diff --git a/packages/sdk/src/flow-extension-submit.ts b/packages/sdk/src/flow-extension-submit.ts index d58f5c8d3..f066b3af4 100644 --- a/packages/sdk/src/flow-extension-submit.ts +++ b/packages/sdk/src/flow-extension-submit.ts @@ -6,7 +6,7 @@ import { extensionManifestOf } from './cli/add-extension.js'; import { assertCompatible, runtimeVersions } from './flow-extension-compat.js'; import type { LoadedAuthoredFlow } from './authored-flow-loader.js'; import type { FlowExtensionManifest } from './flow-extension-manifest.js'; -import type { LoadedFlowExtension } from './flow-extension-loader.js'; +import { probeFlowExtension, type LoadedFlowExtension } from './flow-extension-loader.js'; import { fetchGithubPlugin, MAX_PLUGIN_TOTAL_BYTES, resolveGithubSha, type FetchLike } from './plugin-github.js'; import { PluginError } from './plugin-manifest.js'; import { canonicalPluginRef, parsePluginSource } from './plugin-source.js'; @@ -102,6 +102,12 @@ export async function collectExtensionSubmissions( if (total > MAX_EXTENSIONS_BYTES) { throw new CloudFlowError('invalid_input', `Flow extensions exceed Cloud's ${MAX_EXTENSIONS_BYTES}-byte extensions cap.`); } + try { + for (const submission of submissions) await probeFlowExtension(submission.manifest); + } catch (error) { + if (error instanceof PluginError) throw new CloudFlowError('invalid_input', error.message); + throw error; + } return submissions; } diff --git a/packages/sdk/src/flow-requirements.ts b/packages/sdk/src/flow-requirements.ts index e8edbc82d..7c589e590 100644 --- a/packages/sdk/src/flow-requirements.ts +++ b/packages/sdk/src/flow-requirements.ts @@ -26,9 +26,9 @@ export type FlowHarness = (typeof FLOW_HARNESSES)[number]; export interface FlowIntegrationRequirement { /** Cloud integration provider id (`slack`, `github`, `linear`, …). */ provider: string; - /** `tools`: a header declaration; `source`: a trigger or deploy target; `helper`: body use without a flag, or a YAML helper step. */ - from: 'tools' | 'source' | 'helper' | 'human'; - /** The declaration that requires it, as a reader would name it: `tools.slack`, `--on github`, `f.slack`, `f.human to`. */ + /** Where this need was declared: a header/source/helper/human route, or an extension manifest. */ + from: 'tools' | 'source' | 'helper' | 'human' | 'extension'; + /** The declaration that requires it, as a reader would name it: `tools.slack`, `--on github`, `f.slack`, or `plugin "name"`. */ detail: string; } @@ -56,6 +56,47 @@ export interface FlowRequirementsContext { projectCli?: string; } +export interface FlowExtensionRequirements { + readonly name: string; + readonly permissions: { + readonly integrations: readonly string[]; + readonly harnesses: readonly string[]; + readonly mcp: readonly string[]; + }; +} + +/** Add manifest-only needs without replacing the base flow's first declaration. */ +export function mergeFlowExtensionRequirements( + base: FlowRequirements, + extensions: readonly FlowExtensionRequirements[], +): FlowRequirements { + const integrations = [...base.integrations]; + const integrationNames = new Set(integrations.map(entry => entry.provider)); + const harnessUses = [...base.harnessUses]; + const harnessNames = new Set(base.harnesses); + const mcp = [...base.mcp]; + const mcpNames = new Set(mcp); + + for (const extension of extensions) { + for (const provider of extension.permissions.integrations) { + if (integrationNames.has(provider)) continue; + integrationNames.add(provider); + integrations.push({ provider, from: 'extension', detail: `plugin "${extension.name}"` }); + } + for (const harness of extension.permissions.harnesses) { + if (harnessNames.has(harness)) continue; + harnessNames.add(harness); + harnessUses.push({ harness: harness as FlowHarness, detail: `plugin "${extension.name}"` }); + } + for (const server of extension.permissions.mcp) { + if (mcpNames.has(server)) continue; + mcpNames.add(server); + mcp.push(server); + } + } + return { integrations, harnesses: harnessUses.map(use => use.harness), harnessUses, mcp }; +} + /** The inert subset of an authored definition this module reads. */ export interface RequirementsFlowDefinition { readonly header?: { readonly tools?: Readonly> }; diff --git a/packages/sdk/tests/authored-hooks.test.ts b/packages/sdk/tests/authored-hooks.test.ts index 3787cdaef..342bb3dbc 100644 --- a/packages/sdk/tests/authored-hooks.test.ts +++ b/packages/sdk/tests/authored-hooks.test.ts @@ -78,6 +78,21 @@ describe('hook AND composition', () => { expect(streams.get('hooks')).toHaveLength(1); }); + it('does not persist cancellation as a failed plugin verdict', async () => { + const { journal, streams } = fakeJournal(); + const controller = new AbortController(); + const evaluate = createHookEvaluator({ + journal, rootRunId: 'root-1', flowName: 'software-factory', + declared: ['merge-gate'], + extensions: [extension('first', async () => await new Promise(() => {}))], + signal: controller.signal, + }); + const pending = evaluate('hook-1', 'merge-gate', {}, ctx); + controller.abort(new Error('run cancelled')); + await expect(pending).rejects.toThrow('run cancelled'); + expect(streams.get('hooks')).toBeUndefined(); + }); + it('refuses a hook the base header does not declare', async () => { const { journal } = fakeJournal(); const evaluate = createHookEvaluator({ diff --git a/packages/sdk/tests/bundle.test.ts b/packages/sdk/tests/bundle.test.ts index 8a61478b5..176c1c1c1 100644 --- a/packages/sdk/tests/bundle.test.ts +++ b/packages/sdk/tests/bundle.test.ts @@ -31,7 +31,8 @@ function invoke(args: string[], cwd = repo, env: NodeJS.ProcessEnv = {}) { async function seal(out: string, env: NodeJS.ProcessEnv = { FLOWS_BUILD_KEY: key }, warn = (_: string) => {}) { return sealBundle({ name: 'example', repo: out, out, env, warn, files: [ { path: 'spec.canonical.json', data: canonicalize({ name: 'example' }) }, - { path: 'preflight.json', data: '{}' }, { path: 'lockfile.json', data: '{}' }, + { path: 'preflight.json', data: '{}' }, + { path: 'lockfile.json', data: canonicalize({ version: 2, plugins: [] }) }, ] }); } afterEach(async () => { await Promise.all(temporary.splice(0).map(path => rm(path, { force: true, recursive: true }))); }); diff --git a/packages/sdk/tests/flow-extension-compose.test.ts b/packages/sdk/tests/flow-extension-compose.test.ts index 1b35d9520..922d886d8 100644 --- a/packages/sdk/tests/flow-extension-compose.test.ts +++ b/packages/sdk/tests/flow-extension-compose.test.ts @@ -1,8 +1,9 @@ import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { deployToCloud, parseTriggerSource } from '../src/cloud-deploy.js'; import { collectExtensionSubmissions } from '../src/flow-extension-submit.js'; import { addExtensionPlugin } from '../src/cli/add-extension.js'; import { checkAuthoredTriggers } from '../src/cli/check-triggers.js'; @@ -20,7 +21,11 @@ const REF = `github:AgentWorkforce/flows@${SHA_A}#examples/babysitter`; const versions = { sdk: '2.0.22', surface: '2.0.22' }; const now = () => new Date('2026-09-20T12:00:00Z'); const dirs: string[] = []; -afterEach(() => { dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); }); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + dirs.splice(0).forEach(p => rmSync(p, { recursive: true, force: true })); +}); const BASE = ` import { flow, github } from '@relayflows/surface'; @@ -126,6 +131,7 @@ describe('composing flow extensions onto a base flow', () => { expect(report.ok).toBe(true); expect(report.extensions).toEqual([{ name: 'babysitter', version: '0.1.0', ref: REF, digest: expect.stringMatching(/^[0-9a-f]{64}$/), handlers: 8, hooks: [] }]); expect(report.requirements?.integrations.map(i => i.provider)).toContain('github'); + expect(report.requirements?.harnessUses).toContainEqual({ harness: 'claude', detail: 'plugin "babysitter"' }); expect(await runCli(['check', p.flow], p.io)).toBe(0); expect(p.text()).toContain(`EXTENSION babysitter@0.1.0 ${REF} sha256:`); expect(p.text()).toContain('8 handler(s) composed after the base flow'); @@ -136,6 +142,69 @@ describe('composing flow extensions onto a base flow', () => { expect(submissions[0]!.files.some(f => f.path === 'babysitter.flow.ts' && f.encoding === 'utf8')).toBe(true); expect(submissions[0]!.files.reduce((n, f) => n + f.bytes, 0)).toBeGreaterThan(0); }); + it('flows check probes extension preflight before reporting the project healthy', async () => { + const p = project(); + await install(p, variant(m => ({ + ...m, + preflight: { credentials: ['FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'], servers: [] }, + }))); + const { report } = await checkAuthoredTriggers(p.flow); + expect(report.ok).toBe(false); + expect(report.diagnostics[0]).toMatchObject({ + kind: 'plugin_credential_missing', + message: expect.stringContaining('FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'), + }); + const loaded = await loadAuthoredFlow(p.flow, { versions }); + await expect(collectExtensionSubmissions(loaded)).rejects.toMatchObject({ + code: 'invalid_input', + message: expect.stringContaining('FLOWS_TEST_MISSING_EXTENSION_CREDENTIAL'), + }); + }); + it('uses extension permissions for hosted deploy preflight and the deploy body', async () => { + const p = project(); + await install(p, variant(m => ({ + ...m, + permissions: { ...(m.permissions as object), integrations: ['linear'], harnesses: ['codex'], mcp: ['filesystem'] }, + preflight: { credentials: [], servers: [] }, + }))); + const calls: Array<{ path: string; body: unknown }> = []; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const path = new URL(String(input)).pathname; + calls.push({ path, body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined }); + if (path === '/api/v1/auth/whoami') { + return new Response(JSON.stringify({ currentWorkspace: { id: 'ws-1' } }), { status: 200 }); + } + if (path.endsWith('/integrations/github/status')) return new Response('{"ready":true}', { status: 200 }); + if (path.endsWith('/integrations/linear/status')) return new Response('{"ready":false}', { status: 200 }); + if (path === '/api/v1/flows/deploy') { + return new Response(JSON.stringify({ agentId: 'agent-1', status: 'draft' }), { status: 201 }); + } + return new Response('{}', { status: 404 }); + }); + vi.stubEnv('FLOWS_CLOUD_URL', 'https://cloud-contract.example'); + vi.stubEnv('FLOWS_CLOUD_TOKEN', 'test-token'); + const input = { + path: p.flow, repository: { owner: 'AgentWorkforce', name: 'flows' }, + sources: [parseTriggerSource('github')], approver: 'reviewer', + }; + await expect(deployToCloud(input)).rejects.toMatchObject({ + code: 'integration_not_connected', message: expect.stringContaining('plugin "babysitter"'), + }); + expect(calls.some(call => call.path === '/api/v1/flows/deploy')).toBe(false); + + const deployed = await deployToCloud({ ...input, draft: true }); + const body = calls.findLast(call => call.path === '/api/v1/flows/deploy')!.body; + expect(body).toMatchObject({ + requirements: { integrations: ['github', 'linear'], harnesses: ['codex'], mcp: ['filesystem'] }, + }); + expect(deployed.requirements).toMatchObject({ + integrations: [ + { provider: 'github', from: 'source' }, + { provider: 'linear', from: 'extension', detail: 'plugin "babysitter"' }, + ], + harnesses: ['codex'], mcp: ['filesystem'], + }); + }); }); describe('composition fails closed', () => { diff --git a/packages/sdk/tests/flow-requirements.test.ts b/packages/sdk/tests/flow-requirements.test.ts index 184d38b70..a5ffd7fd3 100644 --- a/packages/sdk/tests/flow-requirements.test.ts +++ b/packages/sdk/tests/flow-requirements.test.ts @@ -6,7 +6,9 @@ import { getFlowDefinition } from '@relayflows/surface/runtime'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runCli } from '../src/cli.js'; import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; -import { describeFlowRequirements, flowRequirements, harnessFromCli } from '../src/flow-requirements.js'; +import { + describeFlowRequirements, flowRequirements, harnessFromCli, mergeFlowExtensionRequirements, +} from '../src/flow-requirements.js'; import { compileSpec } from '../src/compile.js'; import type { FlowSpec } from '../src/spec.js'; @@ -88,6 +90,24 @@ describe('flowRequirements on an authored definition', () => { .toEqual([{ provider: 'slack', from: 'tools', detail: 'tools.slack' }]); }); + it('merges manifest-only extension requirements in declaration order', () => { + const base = flowRequirements(getFlowDefinition(flow('base', { tools: { slack: true } }, async () => {}))); + expect(mergeFlowExtensionRequirements(base, [{ + name: 'babysitter', + permissions: { + integrations: ['slack', 'github'], harnesses: ['codex'], mcp: ['filesystem'], + }, + }])).toEqual({ + integrations: [ + { provider: 'slack', from: 'tools', detail: 'tools.slack' }, + { provider: 'github', from: 'extension', detail: 'plugin "babysitter"' }, + ], + harnesses: ['codex'], + harnessUses: [{ harness: 'codex', detail: 'plugin "babysitter"' }], + mcp: ['filesystem'], + }); + }); + it('reads a compiled spec through its steps and named agents', () => { const spec: FlowSpec = { version: '0.1.0', name: 'yaml', cli: 'gemini', agents: { drafter: { cli: 'codex', model: 'gpt-5' } }, steps: [ { id: 'a', type: 'deterministic', command: 'true' } as never, From be2c441ba0bca4734e0d232039d8e116af4bad1e Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 21 Sep 2026 08:08:42 +0200 Subject: [PATCH 16/17] fix(sdk): recover plugin lock transactions Session-Id: 01a0c26e-544c-7f11-b213-5cd7a2efabfa --- packages/sdk/src/cli/add-extension.ts | 3 +- packages/sdk/src/cli/plugin.ts | 3 +- packages/sdk/src/plugin-lock.ts | 45 ++++++++++++++++++--- packages/sdk/tests/plugin-extension.test.ts | 24 ++++++++++- 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/packages/sdk/src/cli/add-extension.ts b/packages/sdk/src/cli/add-extension.ts index 4f8196255..c53ebeaf2 100644 --- a/packages/sdk/src/cli/add-extension.ts +++ b/packages/sdk/src/cli/add-extension.ts @@ -5,7 +5,7 @@ import { sha256 } from '../bundle.js'; import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow-extension-compat.js'; import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha, type FetchLike, type FetchedPlugin } from '../plugin-github.js'; -import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, writeFlowsAndLock } from '../plugin-lock.js'; +import { PLUGIN_LOCK_FILE, lockWithPlugin, readPluginLock, recoverFlowsAndLock, writeFlowsAndLock } from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; import { PluginError } from '../plugin-manifest.js'; import { canonicalPluginRef, parsePluginSource } from '../plugin-source.js'; @@ -95,6 +95,7 @@ export async function addExtensionPlugin(input: string, io: CliIo, options: AddE const requested = parsePluginSource(input); const root = findPluginProject(options.cwd ?? process.cwd()); if (!root) throw new PluginError('plugin_manifest_invalid', 'flows add requires a project with flows.json.'); + recoverFlowsAndLock(root); const configPath = join(root, 'flows.json'); const config = JSON.parse(readFileSync(configPath, 'utf8')); if (!config || Array.isArray(config) || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every((p: unknown) => typeof p === 'string')))) throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); diff --git a/packages/sdk/src/cli/plugin.ts b/packages/sdk/src/cli/plugin.ts index 085574b4c..c4e234718 100644 --- a/packages/sdk/src/cli/plugin.ts +++ b/packages/sdk/src/cli/plugin.ts @@ -6,7 +6,7 @@ import { assertCompatible, runtimeVersions, type RuntimeVersions } from '../flow import { validateFlowExtensionManifest, type FlowExtensionManifest } from '../flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha, type FetchLike } from '../plugin-github.js'; import { - PLUGIN_LOCK_FILE, lockForDeclared, lockWithPlugin, lockedPlugins, readPluginLock, reconcileDeclaredExtensions, + PLUGIN_LOCK_FILE, lockForDeclared, lockWithPlugin, lockedPlugins, readPluginLock, reconcileDeclaredExtensions, recoverFlowsAndLock, writeFlowsAndLock, type PluginLock, type PluginLockEntry, } from '../plugin-lock.js'; import { findPluginProject } from '../plugin-loader.js'; @@ -120,6 +120,7 @@ export async function runPluginCommand(parsed: PluginArgs, io: CliIo, options: P } function readFlowsConfig(root: string): { path: string; config: Record; plugins: string[] } { + recoverFlowsAndLock(root); const path = join(root, 'flows.json'); let parsed: unknown; try { parsed = JSON.parse(readFileSync(path, 'utf8')); } diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index 63cf67cd7..8a7a4d036 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { PluginError } from './plugin-manifest.js'; import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, type PluginSourceRef } from './plugin-source.js'; @@ -12,6 +12,7 @@ import { SHA, canonicalPluginRef, isGithubPluginRef, parseCanonicalPluginRef, ty */ export const PLUGIN_LOCK_FILE = 'flows.lock.json'; export const PLUGIN_LOCK_VERSION = 2; +const FLOWS_FILE = 'flows.json'; export interface PluginLockEntry { readonly name: string; @@ -63,6 +64,7 @@ export function parsePluginLock(input: unknown): PluginLock { /** Absent file → empty lock; unreadable or malformed → refusal. */ export function readPluginLock(root: string): PluginLock { + recoverFlowsAndLock(root); const path = join(root, PLUGIN_LOCK_FILE); if (!existsSync(path)) return Object.freeze({ version: PLUGIN_LOCK_VERSION, plugins: Object.freeze([]) }); let parsed: unknown; @@ -75,15 +77,47 @@ export function writePluginLock(root: string, lock: PluginLock): void { writeFileSync(join(root, PLUGIN_LOCK_FILE), `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); } -/** Write declaration and lock temps, then rename both so a crash cannot leave one updated. */ +/** + * Finish or abort a two-file update left by a process crash. The lock temp is + * written first: by itself it is only preparation and can be discarded. Once + * the config temp exists, both complete snapshots exist and recovery rolls + * them forward in the same lock-then-declaration order as the writer. + */ +export function recoverFlowsAndLock(root: string, configPath = join(root, FLOWS_FILE)): void { + const jsonTmp = `${configPath}.tmp`; + const lockPath = join(root, PLUGIN_LOCK_FILE); + const lockTmp = `${lockPath}.tmp`; + const hasJson = existsSync(jsonTmp); + const hasLock = existsSync(lockTmp); + if (!hasJson && !hasLock) return; + if (!hasJson) { + unlinkSync(lockTmp); + return; + } + let pendingConfig: unknown; + try { pendingConfig = JSON.parse(readFileSync(jsonTmp, 'utf8')); } + catch { return invalid('pending transaction has invalid flows.json.tmp.'); } + if (!object(pendingConfig) || !Array.isArray(pendingConfig.plugins) + || !pendingConfig.plugins.every(plugin => typeof plugin === 'string')) { + return invalid('pending transaction has invalid flows.json.tmp.'); + } + const lockSource = hasLock ? lockTmp : lockPath; + let pendingLock: unknown; + try { pendingLock = JSON.parse(readFileSync(lockSource, 'utf8')); } + catch { return invalid('pending transaction has no valid lock snapshot.'); } + parsePluginLock(pendingLock); + if (hasLock) renameSync(lockTmp, lockPath); + renameSync(jsonTmp, configPath); +} + +/** Write complete snapshots, then recover them as one roll-forward transaction. */ export function writeFlowsAndLock(root: string, configPath: string, config: Record, plugins: readonly string[], lock: PluginLock): void { const jsonTmp = `${configPath}.tmp`; const lockPath = join(root, PLUGIN_LOCK_FILE); const lockTmp = `${lockPath}.tmp`; - writeFileSync(jsonTmp, `${JSON.stringify({ ...config, plugins: [...plugins] }, null, 2)}\n`); writeFileSync(lockTmp, `${JSON.stringify(parsePluginLock(lock), null, 2)}\n`); - renameSync(lockTmp, lockPath); - renameSync(jsonTmp, configPath); + writeFileSync(jsonTmp, `${JSON.stringify({ ...config, plugins: [...plugins] }, null, 2)}\n`); + recoverFlowsAndLock(root, configPath); } /** @@ -121,6 +155,7 @@ export function lockedPlugins(lock: PluginLock): readonly { ref: string; entry: /** The `github:` entries of `flows.json.plugins`, in declaration order; helper entries are left out. */ export function declaredExtensionRefs(root: string): readonly string[] { + recoverFlowsAndLock(root); let config: { plugins?: unknown }; try { config = JSON.parse(readFileSync(join(root, 'flows.json'), 'utf8')); } catch { throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json.'); } diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts index e112abfc4..b7728daff 100644 --- a/packages/sdk/tests/plugin-extension.test.ts +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -9,7 +9,7 @@ import { runCli } from '../src/cli.js'; import { validateFlowExtensionManifest } from '../src/flow-extension-manifest.js'; import { fetchGithubPlugin, resolveGithubSha } from '../src/plugin-github.js'; import { loadPlugins } from '../src/plugin-loader.js'; -import { parsePluginLock, readPluginLock } from '../src/plugin-lock.js'; +import { parsePluginLock, readPluginLock, reconcileDeclaredExtensions } from '../src/plugin-lock.js'; import { validatePluginManifest } from '../src/plugin-manifest.js'; import { canonicalPluginRef, parseCanonicalPluginRef, parsePluginSource } from '../src/plugin-source.js'; import { pluginStoreDirectory } from '../src/plugin-store.js'; @@ -113,6 +113,28 @@ describe('flows add ', () => { await addExtensionPlugin(REF, c.io, { cwd: c.cwd, fetch: changed.fetch, now, versions }); expect(readPluginLock(c.cwd).plugins[0]!.digest).not.toBe(readPluginLock(a.cwd).plugins[0]!.digest); }); + it('recovers both crash points in the flows.json and lock transaction', async () => { + const gh = github(); + for (const lockAlreadyRenamed of [false, true]) { + const p = project({ cli: 'claude' }); + await addExtensionPlugin(REF, p.io, { cwd: p.cwd, fetch: gh.fetch, now, versions }); + const nextConfig = JSON.stringify({ cli: 'claude', plugins: [] }); + const nextLock = JSON.stringify({ version: 2, plugins: [] }); + writeFileSync(join(p.cwd, 'flows.json.tmp'), nextConfig); + writeFileSync(join(p.cwd, lockAlreadyRenamed ? 'flows.lock.json' : 'flows.lock.json.tmp'), nextLock); + + expect(reconcileDeclaredExtensions(p.cwd)).toEqual([]); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8'))).toEqual({ cli: 'claude', plugins: [] }); + expect(readPluginLock(p.cwd)).toEqual({ version: 2, plugins: [] }); + expect(existsSync(join(p.cwd, 'flows.json.tmp'))).toBe(false); + expect(existsSync(join(p.cwd, 'flows.lock.json.tmp'))).toBe(false); + } + + const aborted = project(); + writeFileSync(join(aborted.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + expect(readPluginLock(aborted.cwd)).toEqual({ version: 2, plugins: [] }); + expect(existsSync(join(aborted.cwd, 'flows.lock.json.tmp'))).toBe(false); + }); it.each([ ['github:AgentWorkforce/flows@nope#examples/babysitter', 'plugin_source_unresolved'], ['github:AgentWorkforce/flows@main#examples/babysitter', 'plugin_source_unresolved'], From b8e5265afbf7c24a254845825fefd6f293fc0dc2 Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 21 Sep 2026 08:17:43 +0200 Subject: [PATCH 17/17] fix(sdk): harden plugin transaction recovery Session-Id: 01a0c26e-544c-7f11-b213-5cd7a2efabfa --- packages/sdk/src/bundle-extensions.ts | 19 +++++++++++++++++-- packages/sdk/src/cli/add.ts | 2 ++ packages/sdk/src/plugin-lock.ts | 17 ++++++++++++++--- packages/sdk/tests/bundle.test.ts | 9 +++++++++ packages/sdk/tests/plugin-extension.test.ts | 18 ++++++++++++++++++ 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/sdk/src/bundle-extensions.ts b/packages/sdk/src/bundle-extensions.ts index 8dd36978c..d008141fc 100644 --- a/packages/sdk/src/bundle-extensions.ts +++ b/packages/sdk/src/bundle-extensions.ts @@ -1,4 +1,4 @@ -import { readFile } from 'node:fs/promises'; +import { readFile, readdir } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import type { BundleFile } from './bundle.js'; import { safePath, sha256 } from './bundle.js'; @@ -36,7 +36,10 @@ export async function verifyBundlePluginLock(bundle: string): Promise { let parsed: unknown; try { parsed = JSON.parse(await readFile(join(bundle, 'lockfile.json'), 'utf8')); } catch { throw new Error('lockfile.json: not valid JSON'); } - if (isLegacyV1Lock(parsed) || isLegacyNpmLock(parsed)) return; + if (isLegacyV1Lock(parsed) || isLegacyNpmLock(parsed)) { + await assertNoLegacyPluginPayload(bundle); + return; + } let lock: PluginLock; try { lock = parsePluginLock(parsed); } catch (error) { @@ -57,6 +60,18 @@ export async function verifyBundlePluginLock(bundle: string): Promise { } } +async function assertNoLegacyPluginPayload(bundle: string): Promise { + let entries: string[]; + try { entries = await readdir(join(bundle, 'plugins')); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + if (entries.length > 0) { + throw new Error('lockfile.json: legacy locks cannot authenticate plugins/ payloads'); + } +} + function isLegacyV1Lock(value: unknown): boolean { return typeof value === 'object' && value !== null && !Array.isArray(value) && (value as { version?: unknown }).version === 1 diff --git a/packages/sdk/src/cli/add.ts b/packages/sdk/src/cli/add.ts index 1dd82c43f..5e6431dbd 100644 --- a/packages/sdk/src/cli/add.ts +++ b/packages/sdk/src/cli/add.ts @@ -3,6 +3,7 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import type { CliIo } from '../cli.js'; import { findPluginProject, probePlugin, readPlugin } from '../plugin-loader.js'; +import { recoverFlowsAndLock } from '../plugin-lock.js'; import { PluginError, pluginPackageName } from '../plugin-manifest.js'; import { isGithubPluginRef } from '../plugin-source.js'; import { addExtensionPlugin, type AddExtensionOptions } from './add-extension.js'; @@ -20,6 +21,7 @@ export async function addPlugin(name: string, io: CliIo, options: { const packageName = pluginPackageName(name); const root = findPluginProject(options.cwd ?? process.cwd()); if (!root) throw new PluginError('plugin_manifest_invalid', 'flows add requires a project with flows.json.'); + recoverFlowsAndLock(root); const configPath = join(root, 'flows.json'); const config = JSON.parse(readFileSync(configPath, 'utf8')); if (!config || Array.isArray(config) || typeof config !== 'object' || (config.plugins !== undefined && (!Array.isArray(config.plugins) || !config.plugins.every((p: unknown) => typeof p === 'string')))) throw new PluginError('plugin_manifest_invalid', 'Invalid flows.json plugins list.'); diff --git a/packages/sdk/src/plugin-lock.ts b/packages/sdk/src/plugin-lock.ts index 8a7a4d036..19fc90495 100644 --- a/packages/sdk/src/plugin-lock.ts +++ b/packages/sdk/src/plugin-lock.ts @@ -96,16 +96,27 @@ export function recoverFlowsAndLock(root: string, configPath = join(root, FLOWS_ } let pendingConfig: unknown; try { pendingConfig = JSON.parse(readFileSync(jsonTmp, 'utf8')); } - catch { return invalid('pending transaction has invalid flows.json.tmp.'); } + catch { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + return invalid('pending transaction has invalid flows.json.tmp.'); + } if (!object(pendingConfig) || !Array.isArray(pendingConfig.plugins) || !pendingConfig.plugins.every(plugin => typeof plugin === 'string')) { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } return invalid('pending transaction has invalid flows.json.tmp.'); } const lockSource = hasLock ? lockTmp : lockPath; let pendingLock: unknown; try { pendingLock = JSON.parse(readFileSync(lockSource, 'utf8')); } - catch { return invalid('pending transaction has no valid lock snapshot.'); } - parsePluginLock(pendingLock); + catch { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + return invalid('pending transaction has no valid lock snapshot.'); + } + try { parsePluginLock(pendingLock); } + catch (error) { + if (hasLock) { unlinkSync(jsonTmp); unlinkSync(lockTmp); return; } + throw error; + } if (hasLock) renameSync(lockTmp, lockPath); renameSync(jsonTmp, configPath); } diff --git a/packages/sdk/tests/bundle.test.ts b/packages/sdk/tests/bundle.test.ts index 176c1c1c1..14e72b6db 100644 --- a/packages/sdk/tests/bundle.test.ts +++ b/packages/sdk/tests/bundle.test.ts @@ -231,6 +231,15 @@ describe('immutable bundles', () => { await verifyBundlePluginLock(bundle); }); + it('never accepts plugin payloads under a legacy npm lock', async () => { + const bundle = await temp(); + await writeFile(join(bundle, 'lockfile.json'), JSON.stringify({ lockfileVersion: 3, packages: {} })); + await verifyBundlePluginLock(bundle); + await mkdir(join(bundle, 'plugins/example'), { recursive: true }); + await writeFile(join(bundle, 'plugins/example/entry.js'), 'export default true;\n'); + await expect(verifyBundlePluginLock(bundle)).rejects.toThrow('legacy locks cannot authenticate plugins/ payloads'); + }); + it('seals materialized flow-extension files under plugins// and verifies them', async () => { const cwd = await temp(); const fixture = join(repo, 'testdata/plugins/extension-babysitter'); diff --git a/packages/sdk/tests/plugin-extension.test.ts b/packages/sdk/tests/plugin-extension.test.ts index b7728daff..47c16c2af 100644 --- a/packages/sdk/tests/plugin-extension.test.ts +++ b/packages/sdk/tests/plugin-extension.test.ts @@ -134,6 +134,14 @@ describe('flows add ', () => { writeFileSync(join(aborted.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); expect(readPluginLock(aborted.cwd)).toEqual({ version: 2, plugins: [] }); expect(existsSync(join(aborted.cwd, 'flows.lock.json.tmp'))).toBe(false); + + const truncated = project({ cli: 'claude' }); + writeFileSync(join(truncated.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + writeFileSync(join(truncated.cwd, 'flows.json.tmp'), '{"plugins":'); + expect(readPluginLock(truncated.cwd)).toEqual({ version: 2, plugins: [] }); + expect(JSON.parse(readFileSync(join(truncated.cwd, 'flows.json'), 'utf8'))).toEqual({ cli: 'claude' }); + expect(existsSync(join(truncated.cwd, 'flows.json.tmp'))).toBe(false); + expect(existsSync(join(truncated.cwd, 'flows.lock.json.tmp'))).toBe(false); }); it.each([ ['github:AgentWorkforce/flows@nope#examples/babysitter', 'plugin_source_unresolved'], @@ -402,6 +410,16 @@ describe('flows plugin remove / update', () => { }); describe('legacy helper plugins are untouched', () => { + it('recovers a pending extension transaction before adding a helper', async () => { + const p = project({ plugins: ['old-helper'] }); + writeFileSync(join(p.cwd, 'flows.lock.json.tmp'), JSON.stringify({ version: 2, plugins: [] })); + writeFileSync(join(p.cwd, 'flows.json.tmp'), JSON.stringify({ plugins: ['pending-helper'] })); + vi.stubEnv('DATADOG_API_KEY', 'test'); + const install = () => cpSync(join(fixtureRoot, 'helper-datadog'), join(p.cwd, 'node_modules/@flows/helper-datadog'), { recursive: true }); + expect(await addPlugin('helper-datadog', p.io, { cwd: p.cwd, install })).toBe(0); + expect(JSON.parse(readFileSync(join(p.cwd, 'flows.json'), 'utf8')).plugins) + .toEqual(['pending-helper', '@flows/helper-datadog']); + }); it('never contacts GitHub for a helper name and leaves the helper path to npm', async () => { const gh = github(); const p = project(); const install = vi.fn(() => { throw { stderr: 'offline' }; });