diff --git a/testdata/shakedown/README.md b/testdata/shakedown/README.md new file mode 100644 index 000000000..cfb3e8345 --- /dev/null +++ b/testdata/shakedown/README.md @@ -0,0 +1,32 @@ +Shakedown flows — 2026-09-10 v2 launch prep + +These are the six canonical scenarios the launch-prep shakedown ran against +`flows run`, extracted from `evidence/shakedown-0910/flow-sources.md` on branch +`shakedown/v2-launch-0910`. Their run transcripts live under +`evidence/shakedown-0910/*.txt` on that branch (drafted at PR #281). + +Scenario map: + +- `hello-world.flow.yaml` — pure deterministic; no external deps. Baseline + that the CLI's happy path works. +- `agent-inline.flow.yaml` — inline `agents: { drafter: { cli, model } }`. + Verifies #263: named-agent inline models work without a `flows.json` in + the current or any ancestor directory. +- `deep-cwd.flow.yaml` — a deterministic flow to run from a working + directory deep enough to push the daemon socket past macOS's SUN_LEN. + Verifies #262: the hashed-under-tmp derivation. +- `chained.flow.yaml` — the flagship `llm → agent → deterministic` shape, + with typed JSON output. **Currently blocked**: no local `llm` worker + path (#273) and no declarative upstream-output binding (#275). +- `error-path.flow.yaml` — deliberately misspells a CLI to exercise the + refusal shape. Sibling `error-dependency.flow.yaml` names a nonexistent + `dependsOn` step. +- `observer.flow.yaml` — a small run intended to be paired with a live + `RELAYCAST_WORKSPACE_KEY` so `flows run` mints an observer URL and prints + it on stdout. Verifies #269 / #286. + +Supplemental variants (agent-codex, chained-codex, chained-llm.flow.ts, +binding, etc.) remain snapshotted at +`evidence/shakedown-0910/flow-sources.md` on the shakedown branch rather +than promoted into this directory: they are exploratory shape-tests, not +canonical scenarios. Promote one if it becomes a first-class example. diff --git a/testdata/shakedown/agent-inline.flow.yaml b/testdata/shakedown/agent-inline.flow.yaml new file mode 100644 index 000000000..c1157b022 --- /dev/null +++ b/testdata/shakedown/agent-inline.flow.yaml @@ -0,0 +1,14 @@ +version: "0.1.0" +name: agent-inline +agents: + drafter: + cli: claude + model: claude-sonnet-4-6 +steps: + - id: draft + type: agent + agent: drafter + instruction: Reply with exactly hello. Do not use tools or modify files. + verification: + type: output_contains + value: hello diff --git a/testdata/shakedown/chained.flow.yaml b/testdata/shakedown/chained.flow.yaml new file mode 100644 index 000000000..141662224 --- /dev/null +++ b/testdata/shakedown/chained.flow.yaml @@ -0,0 +1,30 @@ +version: "0.1.0" +name: chained +agents: + drafter: + cli: claude + model: claude-sonnet-4-6 +steps: + - id: extract + type: llm + cli: claude + model: claude-sonnet-4-6 + prompt: 'Return only JSON: {"message":"hello"}' + output: + type: object + required: [message] + properties: + message: {type: string} + additionalProperties: false + - id: draft + type: agent + agent: drafter + dependsOn: [extract] + instruction: 'Repeat the message from this JSON as plain text: {{steps.extract.output}}. Do not use tools or modify files.' + verification: + type: output_contains + value: hello + - id: write + type: deterministic + dependsOn: [draft] + command: "printf '%s' '{{steps.draft.output}}' > shakedown-result.txt" diff --git a/testdata/shakedown/deep-cwd.flow.yaml b/testdata/shakedown/deep-cwd.flow.yaml new file mode 100644 index 000000000..17135a148 --- /dev/null +++ b/testdata/shakedown/deep-cwd.flow.yaml @@ -0,0 +1,6 @@ +version: "0.1.0" +name: deep-cwd +steps: + - id: hello + type: deterministic + command: printf hello diff --git a/testdata/shakedown/error-dependency.flow.yaml b/testdata/shakedown/error-dependency.flow.yaml new file mode 100644 index 000000000..c9bd9f800 --- /dev/null +++ b/testdata/shakedown/error-dependency.flow.yaml @@ -0,0 +1,7 @@ +version: "0.1.0" +name: error-dependency +steps: + - id: broken + type: deterministic + dependsOn: [does-not-exist] + command: printf hello diff --git a/testdata/shakedown/error-path.flow.yaml b/testdata/shakedown/error-path.flow.yaml new file mode 100644 index 000000000..f66f03630 --- /dev/null +++ b/testdata/shakedown/error-path.flow.yaml @@ -0,0 +1,10 @@ +version: "0.1.0" +name: error-path +steps: + - id: broken + type: agent + cli: claudee + instruction: Reply hello. + verification: + type: output_contains + value: hello diff --git a/testdata/shakedown/hello-world.flow.yaml b/testdata/shakedown/hello-world.flow.yaml new file mode 100644 index 000000000..549b07a47 --- /dev/null +++ b/testdata/shakedown/hello-world.flow.yaml @@ -0,0 +1,6 @@ +version: "0.1.0" +name: hello-world +steps: + - id: hello + type: deterministic + command: printf hello diff --git a/testdata/shakedown/observer.flow.yaml b/testdata/shakedown/observer.flow.yaml new file mode 100644 index 000000000..37569a81e --- /dev/null +++ b/testdata/shakedown/observer.flow.yaml @@ -0,0 +1,6 @@ +version: "0.1.0" +name: observer +steps: + - id: hello + type: deterministic + command: printf hello