From c61cdc4a76dd0744e04acd0f602d66b7def0650c Mon Sep 17 00:00:00 2001 From: Relayflow Date: Tue, 6 Oct 2026 15:21:40 +0000 Subject: [PATCH 1/2] Summarize repeated unprovable-effects warnings in flows check --- docs/SURFACE.md | 10 +- ops/BACKLOG.md | 5 + packages/sdk/src/cli-commands.ts | 3 +- packages/sdk/src/cli-watch.ts | 10 +- packages/sdk/src/cli.ts | 32 ++- packages/sdk/src/cli/diagnostic-fold.ts | 28 +++ .../sdk/tests/check-warning-summary.test.ts | 100 ++++++++ packages/sdk/tests/relay-cli-surface.test.ts | 1 + summary.md | 238 ++++++++++++++++++ testdata/preflight/unprovable-many.flow.yaml | 16 ++ 10 files changed, 426 insertions(+), 17 deletions(-) create mode 100644 packages/sdk/src/cli/diagnostic-fold.ts create mode 100644 packages/sdk/tests/check-warning-summary.test.ts create mode 100644 summary.md create mode 100644 testdata/preflight/unprovable-many.flow.yaml diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 29c6887f3..6f31726b7 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -309,6 +309,14 @@ No process runs between events: the handler wakes, executes to its next await, p deterministic-command preflight gap.” Consequently, `cli_missing` applies to declared `llm` and `agent` CLIs, not deterministic command words. + For YAML/JSON specs, `flows check` summarises `unprovable_effects` warnings + affecting two or more steps in one trailing warning line. The denominator + counts every step in the spec, including `llm` and `agent` steps. + Use `--explain-warnings` (also with `--watch`) to list the original per-step + warnings; `--json` always retains every diagnostic in its stdout report. + Other diagnostic kinds remain individual. `flows run`, `flows resume`, and + `flows deploy` still print individual warnings. + **Project-config discovery:** starting in the flow file's directory, `flows check` walks parent directories through the filesystem root and selects the first readable `flows.json`. That nearest file is the whole project config; it is not merged with outer files. Its schema is `{ "cli"?: , "executors"?: [], "models"?: [], "mcp"?: }`; unknown keys, malformed model entries, and duplicates fail closed as `config_invalid`. A nearer config therefore defines a self-contained nested project boundary and prevents accidental inheritance of outer credentials, executors, or model approvals. The selected path is printed with project-level resolutions and named in refusals; if it declares no `cli` or models, outer configs remain shadowed. At gate 1, a trigger executor is considered registered only when its name is present in this author-written `executors` array; `flows check` does not yet contact a registry, broker, or RelayCron, and absence is `no_executor`. Implementation status for issue #132: this named-agent contract currently @@ -1047,7 +1055,7 @@ verbs: `observer`, which mints an observer link without contacting the daemon, and `status`, which reads a run's journal without one: ```text -flows check [--watch] [--json] +flows check [--watch] [--json] [--explain-warnings] flows run [--json] [--no-spawn] [--data-dir ] flows run [--json] [--no-spawn] [--data-dir ] --input flows resume [--json] [--no-spawn] [--data-dir ] diff --git a/ops/BACKLOG.md b/ops/BACKLOG.md index e70f11b6b..98a4b465b 100644 --- a/ops/BACKLOG.md +++ b/ops/BACKLOG.md @@ -3,6 +3,11 @@ Items the Lead should weigh in assess after ops/DIRECTIVES.md and the current gate's needs. Not commitments; ordering is the Lead's call with evidence. +- **Extend unprovable-effects warning summaries beyond `check`.** + `flows check` folds repeated warnings and offers `--explain-warnings`. + `run`/`resume`/`deploy` retain individual warnings; consider the same rendering + with an appropriate way to request details (run reports have no step total). + - **`timeoutMs` is enforced LATE, not never — a step ran 2.3x past its limit.** Run 6d9bc35c's `verify-1` declares `timeoutMs: 1200000` (20 min) in workflows/drive-cloud.yaml. It ran about 46 minutes — log frozen at 19628 diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index 75a3a0fbd..295f94dad 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -147,7 +147,8 @@ export const CLI_VERBS = [ name: 'check', description: 'Compile and preflight a flow without running it, or opening a daemon socket', args: [{ name: 'source', description: 'flow.ts, flow.yaml, or spec.json', required: true }], - options: [JSON_OPTION, { flags: '--watch', description: 'Re-check on every change to the flow and its imports' }], + options: [JSON_OPTION, { flags: '--watch', description: 'Re-check on every change to the flow and its imports' }, + { flags: '--explain-warnings', description: 'List every unprovable-effects warning per step instead of one summary line' }], variants: ['check'], }, { diff --git a/packages/sdk/src/cli-watch.ts b/packages/sdk/src/cli-watch.ts index 2fe2a6dbb..b126d0026 100644 --- a/packages/sdk/src/cli-watch.ts +++ b/packages/sdk/src/cli-watch.ts @@ -11,7 +11,7 @@ type ExitCode = 0 | 1 | 2 | 3; /** A runner around the ordinary CLI, with a fresh module cache for every check. */ export async function watchCheck( path: string, - json: boolean, + options: { json: boolean; explainWarnings: boolean }, io: CliIo, /** Cancellation, owned by the caller; this function installs no signal handler. */ signal: AbortSignal, @@ -20,8 +20,8 @@ export async function watchCheck( return await watchChecks({ path, signal, - check: () => checkOnce(path, json, io), - clear: () => { if (!json) io.stdout('\x1b[2J\x1b[H'); }, + check: () => checkOnce(path, options, io), + clear: () => { if (!options.json) io.stdout('\x1b[2J\x1b[H'); }, }); } catch (error) { io.stderr(`REFUSED [input_unreadable] Could not watch "${path}": ${error instanceof Error ? error.message : String(error)}`); @@ -140,13 +140,13 @@ function discoverWatchPaths(path: string, imports: Map): Set { +function checkOnce(path: string, options: { json: boolean; explainWarnings: boolean }, io: CliIo): Promise { // Importing an authored TS file in this process again would return its stale // cached definition (including transitive imports). A fresh CLI process uses // precisely the same check pipeline and reporting as a one-shot invocation. const entry = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './cli.ts' : './cli.js', import.meta.url)); return new Promise((resolveCode, reject) => { - const child = spawn(process.execPath, [...process.execArgv, entry, 'check', ...(json ? ['--json'] : []), path], { + const child = spawn(process.execPath, [...process.execArgv, entry, 'check', ...(options.json ? ['--json'] : []), ...(options.explainWarnings ? ['--explain-warnings'] : []), path], { stdio: ['ignore', 'pipe', 'pipe'], // Ctrl-C belongs to the watcher; let an active check finish so its report // and exit status remain a pair, then release all resources. diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 7989f596e..94984fe94 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { foldUnprovableEffects, type FoldOptions } from './cli/diagnostic-fold.js'; import { withSubscriptionMetadata } from './cli/subscription-report.js'; import { addPlugin } from './cli/add.js'; import { parsePluginArgs, runPluginCommand, type PluginArgs } from './cli/plugin.js'; @@ -103,7 +104,7 @@ export type ParsedArgs = | CloudScheduleArgs | { command: 'schedules'; json: boolean } | { command: 'unschedule'; scheduleId: string; json: boolean } - | { command: 'check'; json: boolean; watch: boolean; value: string } + | { command: 'check'; json: boolean; watch: boolean; explainWarnings: boolean; value: string } | { command: 'run'; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'resume'; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'answer'; dataDir: string; json: boolean; spawn: boolean; note: string | undefined; by: string | undefined; runId: string; waitId: string; answer: boolean } @@ -136,7 +137,7 @@ const USAGE = [ 'flows unschedule [--json] ', 'flows deploy @sha256: --to ', 'flows run @sha256: [--bucket ] [--data-dir ] [--json]', - 'flows check [--watch] [--json] ', + 'flows check [--watch] [--json] [--explain-warnings] ', 'flows serve-webhook --data-dir --port

[--allow [,]]', 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--data-dir

] [--local-agent [--agent-capacity ]] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] ', @@ -289,7 +290,7 @@ export async function runCli( if (parsed.command === 'check') { if (parsed.watch) { - return withInterrupt(options.signal, (signal) => watchCheck(parsed.value, parsed.json, io, signal)); + return withInterrupt(options.signal, (signal) => watchCheck(parsed.value, { json: parsed.json, explainWarnings: parsed.explainWarnings }, io, signal)); } // Deliberately daemon-free (kernel/DAEMON-LIFECYCLE.md §4). `checkFlow` is // a compile-and-preflight that opens no daemon socket, and the parser @@ -303,7 +304,7 @@ export async function runCli( const checked = /\.(?:[cm]?[jt]s)$/.test(parsed.value) ? await checkAuthoredFlowComposed(parsed.value) : checkFlow(parsed.value, { warnUnresolvedAgentWorker: true }); - emitCheckReport(checked.report, parsed.json, io); + emitCheckReport(checked.report, parsed.json, io, parsed.explainWarnings); return checked.report.ok ? 0 : 2; } @@ -661,6 +662,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let json = false; let watch = false; + let explainWarnings = false; let cloud = false; let wait = false; let syncCode = false; @@ -707,6 +709,11 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (!isAgentCapacity(agentCapacity)) return undefined; continue; } + if (argument === '--explain-warnings') { + if (command !== 'check' || explainWarnings) return undefined; + explainWarnings = true; + continue; + } if (argument === '--watch') { if (command !== 'check' || watch) return undefined; watch = true; @@ -792,7 +799,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (command === 'run' && input !== undefined && !isAuthoredFlowPath(positionals[0]!)) return undefined; return command === 'check' - ? { command, json, watch, value: positionals[0]! } + ? { command, json, watch, explainWarnings, value: positionals[0]! } : command === 'run' ? { command, bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } : { command, localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; @@ -1056,14 +1063,18 @@ function modelProvenance(source: CliModelSource | undefined): string { return source === undefined ? '' : ` from ${MODEL_PROVENANCE[source]}`; } -function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { +function emitCheckReport(report: CheckReport, json: boolean, io: CliIo, explainWarnings = false): void { + const foldOptions: FoldOptions = { + fold: true, explain: explainWarnings, + ...(report.gates.length > 0 ? { totalSteps: report.gates.length } : {}), + }; if (json) { - emitDiagnostics(report.diagnostics, io); + emitDiagnostics(report.diagnostics, io, foldOptions); io.stdout(JSON.stringify(report)); return; } const deferred = report.diagnostics.filter(isWorkerSurfaceWarning); - emitDiagnostics(report.diagnostics.filter((diagnostic) => !isWorkerSurfaceWarning(diagnostic)), io); + emitDiagnostics(report.diagnostics.filter((diagnostic) => !isWorkerSurfaceWarning(diagnostic)), io, foldOptions); for (const gate of report.gates) { // A gate that accepts every output is legal, but it must not read like a // gate that judges something. @@ -1109,7 +1120,7 @@ function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { // the hosted verbs check the same list against Cloud before submitting. const requires = report.requirements === undefined ? '' : describeFlowRequirements(report.requirements); if (requires) io.stdout(`REQUIRES ${requires}`); - emitDiagnostics(deferred, io); + emitDiagnostics(deferred, io, foldOptions); if (report.ok) io.stdout(`CHECK PASSED ${report.path ?? ''}`.trimEnd()); } @@ -1173,8 +1184,9 @@ function emitRunReport( function emitDiagnostics( diagnostics: CheckReport['diagnostics'] | RunReport['diagnostics'], io: CliIo, + options: FoldOptions = {}, ): void { - for (const diagnostic of diagnostics) { + for (const diagnostic of foldUnprovableEffects(diagnostics, options)) { io.stderr(`${diagnosticLabel(diagnostic.severity)} [${diagnostic.kind}] ${diagnostic.message}`); } } diff --git a/packages/sdk/src/cli/diagnostic-fold.ts b/packages/sdk/src/cli/diagnostic-fold.ts new file mode 100644 index 000000000..d27de6614 --- /dev/null +++ b/packages/sdk/src/cli/diagnostic-fold.ts @@ -0,0 +1,28 @@ +export interface FoldOptions { + /** Collapse repeated warnings only when the caller opts in. */ + fold?: boolean; + explain?: boolean; + /** Every step in the spec, including llm and agent steps. */ + totalSteps?: number; +} + +/** Presentation only: retain the original diagnostics for machine consumers. */ +export function foldUnprovableEffects(diagnostics: readonly D[], options: FoldOptions = {}): readonly (D | { + severity: 'warning'; kind: 'unprovable_effects'; message: string; +})[] { + if (options.fold !== true || options.explain) return diagnostics; + const warnings = diagnostics.filter(d => d.kind === 'unprovable_effects'); + const count = new Set(warnings.flatMap(d => d.stepId === undefined ? [] : [d.stepId])).size + + warnings.filter(d => d.stepId === undefined).length; + if (count < 2) return diagnostics; + const denominator = options.totalSteps === undefined ? '' : ` of ${options.totalSteps}`; + return [ + ...diagnostics.filter(d => d.kind !== 'unprovable_effects'), + { + severity: 'warning', kind: 'unprovable_effects', + message: `${count}${denominator} steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them).`, + }, + ]; +} diff --git a/packages/sdk/tests/check-warning-summary.test.ts b/packages/sdk/tests/check-warning-summary.test.ts new file mode 100644 index 000000000..42ed291a2 --- /dev/null +++ b/packages/sdk/tests/check-warning-summary.test.ts @@ -0,0 +1,100 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { runCli } from '../src/cli.js'; +import { checkFlow } from '../src/cli/check.js'; +import { foldUnprovableEffects } from '../src/cli/diagnostic-fold.js'; + +const fixture = fileURLToPath(new URL('../../../testdata/preflight/unprovable-many.flow.yaml', import.meta.url)); +const warning = (stepId?: string) => ({ severity: 'warning', kind: 'unprovable_effects', message: `effects ${stepId}`, ...(stepId === undefined ? {} : { stepId }) }); +const three = [warning('a'), warning('b'), warning('c')]; +const summary = 'WARNING [unprovable_effects] 3 of 4 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them).'; +const unresolved = 'WARNING [command_unresolved] Step "suspicious" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it.'; +const details = [ + 'WARNING [unprovable_effects] Step "first" command "printf" resolves, but its effects cannot be proven before execution.', + 'WARNING [unprovable_effects] Step "second" command "printf" resolves, but its effects cannot be proven before execution.', + 'WARNING [unprovable_effects] Step "conditional" starts with the shell reserved word "if", whose effects cannot be proven before execution.', +]; +async function invoke(args: string[]) { + const stdout: string[] = [], stderr: string[] = []; + const code = await runCli(args, { stdout: line => stdout.push(line), stderr: line => stderr.push(line) }); + return { stdout, stderr, code }; +} + +describe('warning folding', () => { + it('is opt-in and explain restores the original array', () => { + expect(foldUnprovableEffects(three)).toBe(three); + expect(foldUnprovableEffects(three, {})).toBe(three); + expect(foldUnprovableEffects(three, { fold: true, explain: true })).toBe(three); + }); + it('keeps zero, single, and duplicate-step warnings verbatim', () => { + for (const input of [[], [warning('a')], [warning('a'), warning('a')]]) { + expect(foldUnprovableEffects(input, { fold: true })).toBe(input); + } + }); + it('counts distinct steps and each warning without a step, without mutating input', () => { + const other = { severity: 'warning', kind: 'command_unresolved', message: 'missing' }; + const input = Object.freeze([warning('a'), other, warning('a'), warning(), warning()]); + expect(foldUnprovableEffects(input, { fold: true, totalSteps: 58 })).toEqual([ + other, { severity: 'warning', kind: 'unprovable_effects', message: '3 of 58 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them).' }, + ]); + expect(foldUnprovableEffects(three, { fold: true })[0]?.message).toBe('3 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them).'); + }); +}); + +describe('flows check warning presentation', () => { + it('puts the suspicious command ahead of one summary', async () => { + const result = await invoke(['check', fixture]); + expect(result.code).toBe(0); + expect(result.stderr).toEqual([unresolved, summary]); + }); + it('explains each step in original order', async () => { + const result = await invoke(['check', '--explain-warnings', fixture]); + expect(result.code).toBe(0); + expect(result.stderr).toEqual([...details, unresolved]); + }); + it('keeps JSON byte-identical to the original report, with either text mode', async () => { + const expected = JSON.stringify(checkFlow(fixture, { warnUnresolvedAgentWorker: true }).report); + for (const flags of [[], ['--explain-warnings']]) { + const result = await invoke(['check', '--json', ...flags, fixture]); + expect(result.code).toBe(0); + expect(result.stdout).toEqual([expected]); + expect(result.stderr).toEqual(flags.length ? [...details, unresolved] : [unresolved, summary]); + expect(JSON.parse(result.stdout[0]!).diagnostics.filter((d: { kind: string }) => d.kind === 'unprovable_effects')).toHaveLength(3); + } + }); + it.each([ + ['run', '--explain-warnings', fixture], + ['resume', '--explain-warnings', 'run-id'], + ['check', '--explain-warnings', '--explain-warnings', fixture], + ])('refuses invalid flag use: %j', async (...args) => { + const result = await invoke(args); + expect(result.code).toBe(2); + expect(result.stderr[0]).toContain('REFUSED [invalid_invocation]'); + }); + it('retains every run warning through the run renderer without a daemon', async () => { + const directory = mkdtempSync(join(tmpdir(), 'flows-warning-run-')); + try { + const result = await invoke(['run', '--no-spawn', '--no-observer-link', '--data-dir', directory, fixture]); + expect(result.code).toBe(2); + expect(result.stderr.filter(line => line.startsWith('WARNING [unprovable_effects]'))).toEqual(details); + expect(result.stderr.some(line => line.includes('daemon_unreachable'))).toBe(true); + } finally { rmSync(directory, { recursive: true, force: true }); } + }); + it.each([false, true])('forwards explain=%s to the watch child', async (explain) => { + const entry = fileURLToPath(new URL('../dist/cli.js', import.meta.url)); + const child = spawn(process.execPath, [entry, 'check', '--watch', ...(explain ? ['--explain-warnings'] : []), fixture]); + const closed = once(child, 'close'); + let stdout = '', stderr = ''; + child.stdout.on('data', chunk => { stdout += chunk; }); + child.stderr.on('data', chunk => { stderr += chunk; }); + try { + await expect.poll(() => stdout.includes('CHECK PASSED'), { timeout: 5000 }).toBe(true); + } finally { child.kill('SIGINT'); await closed; } + expect(stderr.trim().split('\n')).toEqual(explain ? [...details, unresolved] : [unresolved, summary]); + }); +}); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index cbb9b27d5..cae3cfea5 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -77,6 +77,7 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'build', argv: ['build', '--verify', '--json', BUNDLE_DIR], variant: 'build' }, { verb: 'build', argv: ['build', BUNDLE_DIR, '--verify'], variant: 'build' }, { verb: 'check', argv: ['check', 'flow.yaml'], variant: 'check' }, + { verb: 'check', argv: ['check', '--explain-warnings', 'flow.yaml'], variant: 'check' }, { verb: 'check', argv: ['check', '--watch', '--json', 'flow.yaml'], variant: 'check' }, // Both `deploy` forms: the positional decides which variant the verb produces. { verb: 'deploy', argv: ['deploy', DIGEST, '--to', 'file:///tmp/bucket'], variant: 'deploy' }, diff --git a/summary.md b/summary.md new file mode 100644 index 000000000..5b99a78a6 --- /dev/null +++ b/summary.md @@ -0,0 +1,238 @@ +`flows check` now collapses repeated `unprovable_effects` warnings into one line, +so a suspicious command remains visible. `--explain-warnings` restores every +original per-step diagnostic, including in watch mode. JSON stdout retains the +original report; only its accompanying stderr is summarised. + +The change follows `reviewed-plan.md`: folding is an explicit renderer opt-in, +starts at two affected steps, counts distinct step IDs, and preserves other +kinds in order before appending the summary. The denominator includes every step. +The ladder examples consequently read `2 of 2`, `2 of 3`, and `2 of 3`. + +Scope: fixes `check` on YAML/JSON specs. The external 58-step spec's format has +not been confirmed. Authored TypeScript checks do not currently produce these +step diagnostics. `run`/`resume`/`deploy` retain their current rendering, with a +follow-up recorded in `ops/BACKLOG.md`. Stdout still prints each `GATE` line. +No effect proving, spec-schema changes, kernel changes, or workflow edits. + +Regression coverage includes opt-in isolation, single-step and duplicate-step +cases, missing step IDs, exact diagnostic ordering, full per-step explanation, +JSON serialization equality, invalid flags, and real watch-child forwarding. +The run-renderer regression uses a deliberate daemon-unreachable invocation +with `--no-spawn`; it verifies warning rendering, not live kernel execution. +The explicit per-step explanation assertion supplies coverage that the existing +ladder prefix assertions no longer provide after folding. + +The `relay-cli-surface.test.ts` change only adds an INVOCATIONS sample for the +newly declared flag. Existing gate assertions remain intact; no covenant-2 +assertion or preflight implementation was edited. + +Verification commands and captured output follow. All final commands below +completed successfully. During implementation, the first build was mistakenly +launched at the repository root (no package.json); the first SDK build found an +import inserted ahead of the CLI shebang (TS18026/TS1005). The import was moved +below the shebang before the successful build and checks captured here. + +```text +$ cd packages/sdk && npm run build + +> @relayflows/sdk@2.0.42 build +> tsc && node scripts/make-cli-executable.mjs + +``` + +```text +$ cd packages/sdk && npm run typecheck + +> @relayflows/sdk@2.0.42 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + +``` + +```text +$ cd packages/sdk && npm run typecheck:tests + +> @relayflows/sdk@2.0.42 typecheck:tests +> tsc -p tsconfig.tests.json + +``` + +```text +$ cd packages/sdk && npx vitest run tests/check-warning-summary.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/check-warning-summary.test.ts (12 tests) 1676ms + ✓ flows check warning presentation > forwards explain=false to the watch child 822ms + ✓ flows check warning presentation > forwards explain=true to the watch child 768ms + + Test Files 1 passed (1) + Tests 12 passed (12) + Start at 15:20:06 + Duration 3.75s (transform 1.24s, setup 13ms, collect 1.89s, tests 1.68s, environment 0ms, prepare 46ms) + +``` + +```text +$ cd packages/sdk && npx vitest run tests/cli.test.ts tests/preflight.test.ts tests/cli-watch.test.ts tests/relay-cli-surface.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/relay-cli-surface.test.ts (88 tests) 105ms + ✓ tests/preflight.test.ts (70 tests) 198ms + ✓ tests/cli.test.ts (71 tests) 3586ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 888ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 534ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 1040ms + ✓ tests/cli-watch.test.ts (10 tests) 16133ms + ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1590ms + ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 1848ms + ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 1809ms + ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 2287ms + ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2358ms + ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1833ms + ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1832ms + ✓ flows check --watch > keeps watching after the target is deleted and recreated 1800ms + ✓ flows check --watch > queues changes during a slow check without overlapping checks 773ms + + Test Files 4 passed (4) + Tests 239 passed (239) + Start at 15:20:21 + Duration 20.60s (transform 2.80s, setup 202ms, collect 10.08s, tests 20.02s, environment 1ms, prepare 323ms) + +``` + +The ten-step demonstration fixture (outside the checkout) was: + +```yaml +version: '0.1.0' +name: many +steps: + - id: s1 + type: deterministic + command: 'node -e "0"' + - id: s2 + type: deterministic + command: 'node -e "0"' + - id: s3 + type: deterministic + command: 'node -e "0"' + - id: s4 + type: deterministic + command: 'node -e "0"' + - id: s5 + type: deterministic + command: 'node -e "0"' + - id: s6 + type: deterministic + command: 'node -e "0"' + - id: s7 + type: deterministic + command: 'node -e "0"' + - id: s8 + type: deterministic + command: 'node -e "0"' + - id: s9 + type: deterministic + command: 'if [ -f x ]; then printf found; fi' + - id: s10 + type: deterministic + command: 'definitely-not-a-real-binary' +``` + +Its sibling `/tmp/foldcheck/flows.json` contains `{}`. + +```text +$ node packages/sdk/dist/cli.js check /tmp/foldcheck/many.flow.yaml +WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +WARNING [unprovable_effects] 9 of 10 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). +GATE step "s1" exit_code from data (kernel, journal-replayable) +GATE step "s2" exit_code from data (kernel, journal-replayable) +GATE step "s3" exit_code from data (kernel, journal-replayable) +GATE step "s4" exit_code from data (kernel, journal-replayable) +GATE step "s5" exit_code from data (kernel, journal-replayable) +GATE step "s6" exit_code from data (kernel, journal-replayable) +GATE step "s7" exit_code from data (kernel, journal-replayable) +GATE step "s8" exit_code from data (kernel, journal-replayable) +GATE step "s9" exit_code from data (kernel, journal-replayable) +GATE step "s10" exit_code from data (kernel, journal-replayable) +CHECK PASSED /tmp/foldcheck/many.flow.yaml +``` + +```text +$ node packages/sdk/dist/cli.js check --explain-warnings /tmp/foldcheck/many.flow.yaml +WARNING [unprovable_effects] Step "s1" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s2" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s3" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s4" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s5" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s6" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s7" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s8" command "node" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "s9" starts with the shell reserved word "if", whose effects cannot be proven before execution. +WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +GATE step "s1" exit_code from data (kernel, journal-replayable) +GATE step "s2" exit_code from data (kernel, journal-replayable) +GATE step "s3" exit_code from data (kernel, journal-replayable) +GATE step "s4" exit_code from data (kernel, journal-replayable) +GATE step "s5" exit_code from data (kernel, journal-replayable) +GATE step "s6" exit_code from data (kernel, journal-replayable) +GATE step "s7" exit_code from data (kernel, journal-replayable) +GATE step "s8" exit_code from data (kernel, journal-replayable) +GATE step "s9" exit_code from data (kernel, journal-replayable) +GATE step "s10" exit_code from data (kernel, journal-replayable) +CHECK PASSED /tmp/foldcheck/many.flow.yaml +``` + +```text +$ node packages/sdk/dist/cli.js check --json /tmp/foldcheck/many.flow.yaml 2>/tmp/warning-json-stderr.log | jq '[.diagnostics[].kind] | group_by(.) | map({(.[0]): length}) | add' +{ + "command_unresolved": 1, + "editor_schema_missing": 1, + "unprovable_effects": 9 +} +``` + +```text +$ node packages/sdk/dist/cli.js check testdata/hello-deterministic.flow.yaml +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +WARNING [unprovable_effects] 2 of 2 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). +GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) +GATE step "shout" exit_code+output_contains from data (kernel, journal-replayable) +CHECK PASSED testdata/hello-deterministic.flow.yaml +``` + +```text +$ node packages/sdk/dist/cli.js check testdata/hello-llm.flow.yaml +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +WARNING [unprovable_effects] 2 of 3 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). +GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) +GATE step "answer" json_schema from data (kernel, journal-replayable) +GATE step "finish" exit_code from data (kernel, journal-replayable) +RESOLVED step "answer" cli "./preflight/authenticated-cli" from project (/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/flows.json) model "deterministic-test-stub" from step +REQUIRES claude (step "answer") +CHECK PASSED testdata/hello-llm.flow.yaml +``` + +```text +$ node packages/sdk/dist/cli.js check testdata/hello-agent.flow.yaml +WARNING [permissions_unenforced] Step "edit" declares permissions (fileGlobs, accessPreset). This declaration is validated and recorded with the step spec; it is not currently enforced (gate 8 / #442). These permissions do not restrict file or network access. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +WARNING [unprovable_effects] 2 of 3 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). +GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) +GATE step "edit" output_contains from data (kernel, journal-replayable) +GATE step "finish" exit_code from data (kernel, journal-replayable) +RESOLVED step "edit" cli "./preflight/authenticated-cli" from project (/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/flows.json) model "test-model-v1" from step +REQUIRES claude (step "edit") +WARNING [agent_worker_unresolved] 1 agent step ("edit") requires an attached worker. For a local run, use `flows run --local-agent `, unless you already attach an agent worker for this daemon. `flows check` does not verify worker attachment: with no worker attached the run parks at the first agent step. +CHECK PASSED testdata/hello-agent.flow.yaml +``` + +JSON stderr captured by the command above: + +```text +WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +WARNING [unprovable_effects] 9 of 10 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). +``` diff --git a/testdata/preflight/unprovable-many.flow.yaml b/testdata/preflight/unprovable-many.flow.yaml new file mode 100644 index 000000000..76b20ed0c --- /dev/null +++ b/testdata/preflight/unprovable-many.flow.yaml @@ -0,0 +1,16 @@ +# yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +version: '0.1.0' +name: unprovable-many +steps: + - id: first + type: deterministic + command: "printf first" + - id: second + type: deterministic + command: "printf second" + - id: conditional + type: deterministic + command: "if [ -f x ]; then printf found; fi" + - id: suspicious + type: deterministic + command: definitely-not-a-real-binary From e08cc123f70bcd3df82629847bab963ff2f153a5 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Wed, 7 Oct 2026 23:36:44 -0700 Subject: [PATCH 2/2] Keep the relayflow working summary out of the change Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 0e078a4c-f89d-4e74-9919-00e1e9a08879 --- summary.md | 238 ----------------------------------------------------- 1 file changed, 238 deletions(-) delete mode 100644 summary.md diff --git a/summary.md b/summary.md deleted file mode 100644 index 5b99a78a6..000000000 --- a/summary.md +++ /dev/null @@ -1,238 +0,0 @@ -`flows check` now collapses repeated `unprovable_effects` warnings into one line, -so a suspicious command remains visible. `--explain-warnings` restores every -original per-step diagnostic, including in watch mode. JSON stdout retains the -original report; only its accompanying stderr is summarised. - -The change follows `reviewed-plan.md`: folding is an explicit renderer opt-in, -starts at two affected steps, counts distinct step IDs, and preserves other -kinds in order before appending the summary. The denominator includes every step. -The ladder examples consequently read `2 of 2`, `2 of 3`, and `2 of 3`. - -Scope: fixes `check` on YAML/JSON specs. The external 58-step spec's format has -not been confirmed. Authored TypeScript checks do not currently produce these -step diagnostics. `run`/`resume`/`deploy` retain their current rendering, with a -follow-up recorded in `ops/BACKLOG.md`. Stdout still prints each `GATE` line. -No effect proving, spec-schema changes, kernel changes, or workflow edits. - -Regression coverage includes opt-in isolation, single-step and duplicate-step -cases, missing step IDs, exact diagnostic ordering, full per-step explanation, -JSON serialization equality, invalid flags, and real watch-child forwarding. -The run-renderer regression uses a deliberate daemon-unreachable invocation -with `--no-spawn`; it verifies warning rendering, not live kernel execution. -The explicit per-step explanation assertion supplies coverage that the existing -ladder prefix assertions no longer provide after folding. - -The `relay-cli-surface.test.ts` change only adds an INVOCATIONS sample for the -newly declared flag. Existing gate assertions remain intact; no covenant-2 -assertion or preflight implementation was edited. - -Verification commands and captured output follow. All final commands below -completed successfully. During implementation, the first build was mistakenly -launched at the repository root (no package.json); the first SDK build found an -import inserted ahead of the CLI shebang (TS18026/TS1005). The import was moved -below the shebang before the successful build and checks captured here. - -```text -$ cd packages/sdk && npm run build - -> @relayflows/sdk@2.0.42 build -> tsc && node scripts/make-cli-executable.mjs - -``` - -```text -$ cd packages/sdk && npm run typecheck - -> @relayflows/sdk@2.0.42 typecheck -> tsc --noEmit && tsc -p tsconfig.type-tests.json - -``` - -```text -$ cd packages/sdk && npm run typecheck:tests - -> @relayflows/sdk@2.0.42 typecheck:tests -> tsc -p tsconfig.tests.json - -``` - -```text -$ cd packages/sdk && npx vitest run tests/check-warning-summary.test.ts - - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk - - ✓ tests/check-warning-summary.test.ts (12 tests) 1676ms - ✓ flows check warning presentation > forwards explain=false to the watch child 822ms - ✓ flows check warning presentation > forwards explain=true to the watch child 768ms - - Test Files 1 passed (1) - Tests 12 passed (12) - Start at 15:20:06 - Duration 3.75s (transform 1.24s, setup 13ms, collect 1.89s, tests 1.68s, environment 0ms, prepare 46ms) - -``` - -```text -$ cd packages/sdk && npx vitest run tests/cli.test.ts tests/preflight.test.ts tests/cli-watch.test.ts tests/relay-cli-surface.test.ts - - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk - - ✓ tests/relay-cli-surface.test.ts (88 tests) 105ms - ✓ tests/preflight.test.ts (70 tests) 198ms - ✓ tests/cli.test.ts (71 tests) 3586ms - ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 888ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 534ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 1040ms - ✓ tests/cli-watch.test.ts (10 tests) 16133ms - ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1590ms - ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 1848ms - ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 1809ms - ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 2287ms - ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2358ms - ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1833ms - ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1832ms - ✓ flows check --watch > keeps watching after the target is deleted and recreated 1800ms - ✓ flows check --watch > queues changes during a slow check without overlapping checks 773ms - - Test Files 4 passed (4) - Tests 239 passed (239) - Start at 15:20:21 - Duration 20.60s (transform 2.80s, setup 202ms, collect 10.08s, tests 20.02s, environment 1ms, prepare 323ms) - -``` - -The ten-step demonstration fixture (outside the checkout) was: - -```yaml -version: '0.1.0' -name: many -steps: - - id: s1 - type: deterministic - command: 'node -e "0"' - - id: s2 - type: deterministic - command: 'node -e "0"' - - id: s3 - type: deterministic - command: 'node -e "0"' - - id: s4 - type: deterministic - command: 'node -e "0"' - - id: s5 - type: deterministic - command: 'node -e "0"' - - id: s6 - type: deterministic - command: 'node -e "0"' - - id: s7 - type: deterministic - command: 'node -e "0"' - - id: s8 - type: deterministic - command: 'node -e "0"' - - id: s9 - type: deterministic - command: 'if [ -f x ]; then printf found; fi' - - id: s10 - type: deterministic - command: 'definitely-not-a-real-binary' -``` - -Its sibling `/tmp/foldcheck/flows.json` contains `{}`. - -```text -$ node packages/sdk/dist/cli.js check /tmp/foldcheck/many.flow.yaml -WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -WARNING [unprovable_effects] 9 of 10 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). -GATE step "s1" exit_code from data (kernel, journal-replayable) -GATE step "s2" exit_code from data (kernel, journal-replayable) -GATE step "s3" exit_code from data (kernel, journal-replayable) -GATE step "s4" exit_code from data (kernel, journal-replayable) -GATE step "s5" exit_code from data (kernel, journal-replayable) -GATE step "s6" exit_code from data (kernel, journal-replayable) -GATE step "s7" exit_code from data (kernel, journal-replayable) -GATE step "s8" exit_code from data (kernel, journal-replayable) -GATE step "s9" exit_code from data (kernel, journal-replayable) -GATE step "s10" exit_code from data (kernel, journal-replayable) -CHECK PASSED /tmp/foldcheck/many.flow.yaml -``` - -```text -$ node packages/sdk/dist/cli.js check --explain-warnings /tmp/foldcheck/many.flow.yaml -WARNING [unprovable_effects] Step "s1" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s2" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s3" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s4" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s5" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s6" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s7" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s8" command "node" resolves, but its effects cannot be proven before execution. -WARNING [unprovable_effects] Step "s9" starts with the shell reserved word "if", whose effects cannot be proven before execution. -WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -GATE step "s1" exit_code from data (kernel, journal-replayable) -GATE step "s2" exit_code from data (kernel, journal-replayable) -GATE step "s3" exit_code from data (kernel, journal-replayable) -GATE step "s4" exit_code from data (kernel, journal-replayable) -GATE step "s5" exit_code from data (kernel, journal-replayable) -GATE step "s6" exit_code from data (kernel, journal-replayable) -GATE step "s7" exit_code from data (kernel, journal-replayable) -GATE step "s8" exit_code from data (kernel, journal-replayable) -GATE step "s9" exit_code from data (kernel, journal-replayable) -GATE step "s10" exit_code from data (kernel, journal-replayable) -CHECK PASSED /tmp/foldcheck/many.flow.yaml -``` - -```text -$ node packages/sdk/dist/cli.js check --json /tmp/foldcheck/many.flow.yaml 2>/tmp/warning-json-stderr.log | jq '[.diagnostics[].kind] | group_by(.) | map({(.[0]): length}) | add' -{ - "command_unresolved": 1, - "editor_schema_missing": 1, - "unprovable_effects": 9 -} -``` - -```text -$ node packages/sdk/dist/cli.js check testdata/hello-deterministic.flow.yaml -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -WARNING [unprovable_effects] 2 of 2 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). -GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) -GATE step "shout" exit_code+output_contains from data (kernel, journal-replayable) -CHECK PASSED testdata/hello-deterministic.flow.yaml -``` - -```text -$ node packages/sdk/dist/cli.js check testdata/hello-llm.flow.yaml -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -WARNING [unprovable_effects] 2 of 3 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). -GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) -GATE step "answer" json_schema from data (kernel, journal-replayable) -GATE step "finish" exit_code from data (kernel, journal-replayable) -RESOLVED step "answer" cli "./preflight/authenticated-cli" from project (/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/flows.json) model "deterministic-test-stub" from step -REQUIRES claude (step "answer") -CHECK PASSED testdata/hello-llm.flow.yaml -``` - -```text -$ node packages/sdk/dist/cli.js check testdata/hello-agent.flow.yaml -WARNING [permissions_unenforced] Step "edit" declares permissions (fileGlobs, accessPreset). This declaration is validated and recorded with the step spec; it is not currently enforced (gate 8 / #442). These permissions do not restrict file or network access. -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -WARNING [unprovable_effects] 2 of 3 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). -GATE step "greet" exit_code+output_contains from data (kernel, journal-replayable) -GATE step "edit" output_contains from data (kernel, journal-replayable) -GATE step "finish" exit_code from data (kernel, journal-replayable) -RESOLVED step "edit" cli "./preflight/authenticated-cli" from project (/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/flows.json) model "test-model-v1" from step -REQUIRES claude (step "edit") -WARNING [agent_worker_unresolved] 1 agent step ("edit") requires an attached worker. For a local run, use `flows run --local-agent `, unless you already attach an agent worker for this daemon. `flows check` does not verify worker attachment: with no worker attached the run parks at the first agent step. -CHECK PASSED testdata/hello-agent.flow.yaml -``` - -JSON stderr captured by the command above: - -```text -WARNING [command_unresolved] Step "s10" command "definitely-not-a-real-binary" does not resolve as an executable; it runs only if the shell supplies it. -WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json -WARNING [unprovable_effects] 9 of 10 steps have effects that cannot be proven before execution (rerun with --explain-warnings to list them). -```