From 353417a1774b12d9666a2410d3df2982af8c7deb Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:29:03 +0200 Subject: [PATCH 01/41] fix(workflow): record bad_session in run records --- src/__tests__/workflow-records.test.ts | 2 +- src/workflow/records.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index a16a070f1..b8c79b439 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -203,7 +203,7 @@ describe('#1083 workflow run records', () => { it('publishes the failure code of every step failure teamai writes, and only codes it can name', () => { // Codes a `code: '…'` literal in src/workflow writes that never become a run's failure. - const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_session', 'bad_workspace'] satisfies Array; + const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace'] satisfies Array; const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'busy']; const source = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'workflow'); const written = new Set(fs.readdirSync(source).filter((file) => file.endsWith('.ts')) diff --git a/src/workflow/records.ts b/src/workflow/records.ts index 243aad253..85124560d 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -34,7 +34,7 @@ const RECORDED_FAILURES = [ 'cancelled', 'hitl_required', 'loop_exhausted', 'invalid_output', 'invalid_template', 'invalid_inputs', 'invalid_items', 'item_cycle', 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', - 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', + 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; From 550eb7e585d4f3a47bce16e5a7c94975b0163d7d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:31:47 +0200 Subject: [PATCH 02/41] fix(workflow): reset every tool's agent configuration to the run base --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 34 ++++++++++++++++++++++++ src/workflow/worktree.ts | 21 ++++++++++++++- 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index ea816a895..69cbf08cb 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It waits, and does not hand the lease out, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It fails `workspace_base` when `continue` has no enclosing `new`, or when `next` sent no `base`. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENTS.md`, `CLAUDE.md`, `.claude`, `.codex`, `.opencode`, `.cursor`, `.agents`, and `.githooks` with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It waits, and does not hand the lease out, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It fails `workspace_base` when `continue` has no enclosing `new`, or when `next` sent no `base`. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 48bc3f263..4025e8a2f 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -143,7 +143,7 @@ A command with no `workspace` is read-only, even inside a workspace a parent ope `workspace: continue` stays on the branch an enclosing `workflow` step opened with `new`. A prompt with no `workspace` continues that copy. A `for_each` with `workspace: new` gives each item its own branch, so items do not share a worktree. `steps..branches` lists those branches, and each `results` entry has `branch` when the item has one. An item with `after` starts from the branches it waits for. A top-level `workspace: continue`, with no enclosing `new`, fails the step with `workspace_base`. -The step sees the agent files from the commit the run started at: `AGENTS.md`, `CLAUDE.md`, `.claude`, `.codex`, `.opencode`, `.cursor`, `.agents`, and `.githooks`. A hook an earlier step committed does not run in a later step. The team's skills, rules, agents and MCP are then delivered into the worktree, as they are for any new worktree of this checkout. Those delivered files are not part of the step's commit. A file the step itself edits is committed. +The step sees the agent files from the commit the run started at: `AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, every tool's project directory such as `.claude` or `.tclaude`, the project MCP files `.mcp.json` and `opencode.json`, and Copilot's agent paths under `.github`. A hook an earlier step committed does not run in a later step. The team's skills, rules, agents and MCP are then delivered into the worktree, as they are for any new worktree of this checkout. Those delivered files are not part of the step's commit. A file the step itself edits is committed. When the checkout has an `origin` and the step made commits, the driver pushes that branch. A later attempt that adds no commit still pushes when this branch has commits `origin` does not have. That includes a first push that never landed, and an origin that has no branches yet. A failed push fails the step with `push_failed` and keeps the worktree. With no `origin`, or when `origin` already has that commit, the run still succeeds and reports the branch. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index dcb94e4a4..e423c30ae 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -143,7 +143,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom `workspace: continue` 留在外层 `workflow` 步骤用 `new` 打开的那条分支上。没有 `workspace` 的提示词会沿用这份副本。带 `workspace: new` 的 `for_each` 给每一项自己的分支，因此各项不共用一个 worktree。`steps..branches` 列出这些分支；某一项有分支时，对应的 `results` 条目带 `branch`。带 `after` 的项从它等待的那些分支开始。顶层的 `workspace: continue` 没有外层 `new` 时，步骤以 `workspace_base` 失败。 -步骤看到的是运行开始时那个提交上的 agent 文件：`AGENTS.md`、`CLAUDE.md`、`.claude`、`.codex`、`.opencode`、`.cursor`、`.agents` 和 `.githooks`。前一个步骤提交的 hook 不会在后面的步骤里运行。团队的 skills、rules、agents 和 MCP 随后会交付到这个 worktree，和这个检出目录的其他新 worktree 一样。这些交付的文件不属于该步骤的提交。步骤自己改过的文件会被提交。 +步骤看到的是运行开始时那个提交上的 agent 文件：`AGENTS.md`、`CLAUDE.md`、`.agents`、`.githooks`，每个工具的项目目录（如 `.claude` 或 `.tclaude`），项目级 MCP 文件 `.mcp.json` 和 `opencode.json`，以及 Copilot 在 `.github` 下的 agent 路径。前一个步骤提交的 hook 不会在后面的步骤里运行。团队的 skills、rules、agents 和 MCP 随后会交付到这个 worktree，和这个检出目录的其他新 worktree 一样。这些交付的文件不属于该步骤的提交。步骤自己改过的文件会被提交。 检出目录有 `origin`，并且这一步产生了提交时，驱动会推送该分支。后一次尝试即使没有新提交，只要这个分支上有 `origin` 还没有的提交，仍会再推一次。这包括第一次推送根本没成功，以及 `origin` 上还没有任何分支。推送失败时步骤以 `push_failed` 失败，并保留 worktree。没有 `origin`，或 `origin` 已经有这个提交时，运行仍然成功，并报告该分支。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index b8a93e07a..ce6eda009 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -527,6 +527,40 @@ steps: expect(fs.existsSync(worktreePath(workspace, branch!))).toBe(false); }); + it('runs a later step on the base MCP and tool settings an earlier step committed, and keeps its own edit', async () => { + const files = ['.mcp.json', 'opencode.json', '.tclaude/settings.json']; + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A later step does not see MCP or tool settings committed earlier. +steps: + - id: work + workspace: new + workflow: + steps: + - id: plant + workspace: continue + command: node -e "const fs=require('fs'); for (const f of ${JSON.stringify(files).replace(/"/g, "'")}) fs.writeFileSync(f,'earlier')" + - id: check + needs: [plant] + workspace: continue + command: node -e "const fs=require('fs'); for (const f of ${JSON.stringify(files).replace(/"/g, "'")}) if (fs.readFileSync(f,'utf8')!=='base') process.exit(2); fs.writeFileSync('.mcp.json','later')" +`); + fs.mkdirSync(path.join(workspace, '.tclaude')); + for (const file of files) fs.writeFileSync(path.join(workspace, file), 'base'); + execFileSync('git', ['add', ...files], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'tool config'], { cwd: workspace }); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await readRun(config, runId); + const check = typeof run === 'object' ? run.steps.find((step) => step.id === 'work.check') : undefined; + const branch = check && 'changes' in check ? check.changes?.branch : undefined; + expect(git(['show', `${branch}:.mcp.json`], workspace)).toBe('later'); + expect(git(['show', `${branch}:opencode.json`], workspace)).toBe('earlier'); + expect(git(['show', `${branch}:.tclaude/settings.json`], workspace)).toBe('earlier'); + }); + it('delivers a team skill into a new workspace and leaves it out of the branch', async () => { const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-workspace-'))); dirs.push(root); diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 42ae527ea..29c76b48f 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -8,8 +8,27 @@ import { execFile, spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { TeamaiConfigBaseSchema } from '../types.js'; -const AGENT_PATHS = ['AGENTS.md', 'CLAUDE.md', '.claude', '.codex', '.opencode', '.cursor', '.agents', '.githooks']; +/** Agent configuration a workspace step reads from the run's base rather than from its branch. */ +export const AGENT_PATHS: readonly string[] = agentConfigPaths(); + +function agentConfigPaths(): string[] { + const paths = new Set(['AGENTS.md', 'CLAUDE.md', '.agents', '.githooks']); + for (const tool of Object.values(TeamaiConfigBaseSchema.shape.toolPaths.parse(undefined))) { + for (const field of ['skills', 'rules', 'settings', 'hooks', 'claudemd', 'agents', 'mcpProject'] as const) { + const relative = tool[field]; + if (relative) paths.add(configRoot(field === 'hooks' ? path.posix.dirname(relative) : relative)); + } + } + return [...paths]; +} + +/** A tool's own directory is reset whole. `.github` also holds workflows and code owners, so only its agent paths are. */ +function configRoot(relative: string): string { + const first = relative.split('/')[0] ?? relative; + return first !== relative && first.startsWith('.') && first !== '.github' ? first : relative; +} export interface GitResult { code: number; From 0866894779080b92a8fcbac21d907904d9cfa5de Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:34:53 +0200 Subject: [PATCH 03/41] fix(workflow): reject reported changes that do not match the lease --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 26 ++++++++++++++++++++++++ src/workflow/engine.ts | 15 +++++++++++++- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 4025e8a2f..edc713af7 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -129,7 +129,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. +`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. The run is stored in this scope's TeamAI data directory. The command exits when it finishes. A local run that ends `failed` or `cancelled` exits with status 1. No workflow process keeps running. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index e423c30ae..95b44d187 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -129,7 +129,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 +`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 运行状态保存在当前 scope 的 TeamAI 数据目录里。命令结束后进程就退出。本地运行以 `failed` 或 `cancelled` 结束时，退出码是 1。不会留下常驻的工作流进程。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index ce6eda009..6fe284bec 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -417,6 +417,32 @@ steps: expect(other).toMatchObject({ data: { wait: { reason: expect.stringContaining('another driver') } } }); }); + it('rejects a report whose changes name another branch or base than the lease, and keeps the step leased', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Changes must match the lease. +steps: + - id: edit + workspace: new + command: echo edit +`); + const owner = driver(workspace); + const base = { ref: 'HEAD', commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const leased = await leaseNext(config, runId, owner, { base, workspaceModes: ['read-only', 'new', 'continue'] }); + const lease = 'data' in leased ? leased.data.lease as { attempt: string; revision: number; workspace?: { branch: string } } : undefined; + if (!lease?.workspace) throw new Error(JSON.stringify(leased)); + const changes = { branch: lease.workspace.branch, base: base.commit, from: base.commit, to: base.commit, commits: [] }; + const report = (override: Partial) => reportAttempt(config, runId, { + attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver: owner, changes: { ...changes, ...override }, + }); + + expect(await report({ branch: 'teamai/elsewhere' })).toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining('teamai/elsewhere') } }); + expect(await report({ base: 'f'.repeat(40) })).toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining('f'.repeat(40)) } }); + const run = await readRun(config, runId); + expect(typeof run === 'object' ? run.steps.find((step) => step.id === 'edit') : undefined).toMatchObject({ status: 'leased', lease: { attempt: lease.attempt } }); + expect(await report({})).toMatchObject({ data: { accepted: true, cleanup: true } }); + }); + it('merges a parent dependency into the first child that continues its copy', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 4b0d00752..993010637 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1179,7 +1179,20 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: return { save: false, envelope: error('wrong_driver', 'This lease belongs to a different driver') }; } - const defined = locateStep(run, step.id)?.step; + const located = locateStep(run, step.id); + const defined = located?.step; + if (report.changes) { + const contract = located && planWorkspace(run, located.scope, step, located.step, { + driver: step.lease.driver, modes: ['read-only', 'new', 'continue'], + }, true).contract; + if (!contract || report.changes.branch !== contract.branch || report.changes.base !== contract.base.commit) { + const leased = contract ? `branch ${contract.branch} on base ${contract.base.commit}` : 'no workspace'; + return { + save: false, + envelope: error('invalid_report', `changes name branch ${report.changes.branch} on base ${report.changes.base}, but attempt ${report.attempt} was leased ${leased}`), + }; + } + } let outcome = report.outcome; let message = report.message; let failure = report.failure; From d254464b21f3ac74bbd482cecb5a6e75a773b8dc Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:40:05 +0200 Subject: [PATCH 04/41] fix(workflow): record the run base at workflow start The base came from whichever driver first sent one to next, with ref always HEAD. A missing ref then made the local driver drop the whole workspace and run the step in the member's checkout. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-commands.test.ts | 56 ++++++++++++++++++++++++ src/__tests__/workflow-workspace.test.ts | 50 ++++++++++++++++++++- src/workflow-commands.ts | 4 +- src/workflow/engine.ts | 14 +++--- src/workflow/local-driver.ts | 12 +++-- src/workflow/types.ts | 9 +++- src/workflow/workspace.ts | 4 +- src/workflow/worktree.ts | 7 ++- 9 files changed, 140 insertions(+), 18 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 69cbf08cb..4cb8a35cc 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It waits, and does not hand the lease out, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It fails `workspace_base` when `continue` has no enclosing `new`, or when `next` sent no `base`. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It waits, and does not hand the lease out, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-commands.test.ts b/src/__tests__/workflow-commands.test.ts index 53f8cd4a1..a207a3213 100644 --- a/src/__tests__/workflow-commands.test.ts +++ b/src/__tests__/workflow-commands.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -20,6 +21,7 @@ vi.mock('../config.js', async (importOriginal) => { import { setStderrOnly } from '../utils/logger.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; import { startRun } from '../workflow/engine.js'; +import { readRun } from '../workflow/store.js'; import { workflowNext, workflowReport, workflowStart } from '../workflow-commands.js'; const dirs: string[] = []; @@ -72,6 +74,7 @@ async function capture(run: () => Promise): Promise<{ data?: Record { + vi.restoreAllMocks(); state.config = undefined; setStderrOnly(false); process.exitCode = undefined; @@ -127,6 +130,8 @@ describe('workflow json payloads', () => { ['running that is not an array', { running: 'a00000000' }, 'running must be an array of attempt ids'], ['running that holds a number', { running: [1] }, 'running must be an array of attempt ids'], ['canTakeLease that is not a boolean', { canTakeLease: 'false' }, 'canTakeLease must be a boolean'], + ['a base without a commit', { base: { ref: 'main' } }, 'base must include commit, and ref when it is set must not be empty'], + ['a base with an empty ref', { base: { ref: '', commit: 'c0ffee' } }, 'base must include commit, and ref when it is set must not be empty'], ])('rejects a next payload with %s', async (_case, fields, message) => { const { config, runId } = await startBuild(); state.config = config; @@ -183,3 +188,54 @@ steps: }); }); }); + +describe('workflow start base', () => { + function checkout(): { repo: string; commit: string } { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-workflow-start-')); + dirs.push(repo); + const git = (...args: string[]): string => execFileSync('git', args, { cwd: repo, encoding: 'utf8' }).trim(); + git('init', '-q', '--initial-branch=feature/start'); + fs.writeFileSync(path.join(repo, 'tracked.txt'), 'clean\n'); + git('add', 'tracked.txt'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'initial'); + return { repo, commit: git('rev-parse', 'HEAD') }; + } + + async function startIn(repo: string): Promise<{ config: LocalConfig; runId: string }> { + const config = testConfig(); + fs.writeFileSync(path.join(config.repo.localPath, 'workflows', 'change.yaml'), `schemaVersion: v1alpha1 +name: change +description: One workspace step. +steps: + - id: edit + workspace: new + command: node edit.js +`); + await syncWorkflowDefinitions(config); + state.config = config; + vi.spyOn(process, 'cwd').mockReturnValue(repo); + const started = await capture(() => workflowStart('change', {})); + return { config, runId: String(started.data?.run) }; + } + + it('records the branch and commit the checkout was on, and a later base from another driver does not change them', async () => { + const { repo, commit } = checkout(); + const { config, runId } = await startIn(repo); + + const other = { kind: 'ide', machine: 'b', workspace: '/elsewhere', can: { workspace: ['read-only', 'new', 'continue'] } }; + const leased = await capture(() => workflowNext(runId, { json: JSON.stringify({ driver: other, base: { commit: 'f'.repeat(40) } }) })); + + expect(leased).toMatchObject({ data: { lease: { step: 'edit', workspace: { base: { ref: 'feature/start', commit } } } } }); + const run = await readRun(config, runId); + expect(typeof run === 'object' ? run.base : run).toEqual({ ref: 'feature/start', commit }); + }); + + it('records no ref when the checkout has a detached HEAD', async () => { + const { repo, commit } = checkout(); + execFileSync('git', ['checkout', '-q', '--detach'], { cwd: repo }); + const { config, runId } = await startIn(repo); + + const run = await readRun(config, runId); + expect(typeof run === 'object' ? run.base : run).toEqual({ commit }); + }); +}); diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 6fe284bec..54fde61ac 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -6,12 +6,13 @@ import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { detectProjectConfig, loadStateForScope } from '../config.js'; import { resolveAnchors } from '../utils/git.js'; import { resolvePartitionDir } from '../utils/partition.js'; import type { LocalConfig } from '../types.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; +import * as engine from '../workflow/engine.js'; import { leaseNext, reportAttempt, startRun } from '../workflow/engine.js'; import { runLocalDriver } from '../workflow/local-driver.js'; import { cancelRun, retryStep } from '../workflow/run-controls.js'; @@ -58,6 +59,7 @@ function git(args: string[], cwd: string): string { const driver = (workspace: string, machine = 'host'): DriverIdentity => ({ kind: 'local', machine, workspace }); afterEach(() => { + vi.restoreAllMocks(); for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); @@ -417,6 +419,52 @@ steps: expect(other).toMatchObject({ data: { wait: { reason: expect.stringContaining('another driver') } } }); }); + it('runs a workspace step from a detached HEAD in its own worktree', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A detached checkout still opens a workspace. +steps: + - id: edit + workspace: new + command: node -e "require('fs').writeFileSync('edited.txt','yes')" +`); + git(['checkout', '-q', '--detach'], workspace); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await readRun(config, runId); + const edit = typeof run === 'object' ? run.steps.find((step) => step.id === 'edit') : undefined; + const branch = edit && 'changes' in edit ? edit.changes?.branch : undefined; + expect(git(['show', `${branch}:edited.txt`], workspace)).toBe('yes'); + expect(fs.existsSync(path.join(workspace, 'edited.txt'))).toBe(false); + }); + + it('fails a step whose lease carries a workspace this driver cannot read, and does not run it in the checkout', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A newer workspace contract. +steps: + - id: edit + workspace: new + command: node -e "require('fs').writeFileSync('edited.txt','yes')" +`); + const leaseNextAsSent = engine.leaseNext; + vi.spyOn(engine, 'leaseNext').mockImplementation(async (...args) => { + const answer = await leaseNextAsSent(...args); + const lease = 'data' in answer ? answer.data.lease : undefined; + if (lease && typeof lease === 'object' && 'workspace' in lease) Object.assign(lease, { workspace: { mode: 'clone' } }); + return answer; + }); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await readRun(config, runId); + expect(typeof run === 'object' ? run.steps : run).toMatchObject([{ id: 'edit', status: 'failed', failure: { code: 'workspace_base' } }]); + expect(fs.existsSync(path.join(workspace, 'edited.txt'))).toBe(false); + }); + it('rejects a report whose changes name another branch or base than the lease, and keeps the step leased', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow-commands.ts b/src/workflow-commands.ts index 3ad2018d1..83b76f82b 100644 --- a/src/workflow-commands.ts +++ b/src/workflow-commands.ts @@ -5,6 +5,7 @@ import { cancelRun, respondToHitl, retryStep } from './workflow/run-controls.js' import { localDriver, runLocalDriver } from './workflow/local-driver.js'; import { scopedToolPaths, type LocalConfig, type TeamaiConfig } from './types.js'; import { isRunId, readRun } from './workflow/store.js'; +import { headCommit } from './workflow/worktree.js'; import type { WorkflowEnvelope } from './workflow/types.js'; function writeEnvelope(envelope: WorkflowEnvelope): void { @@ -93,7 +94,8 @@ export async function workflowStart(name: string, options: { driver?: string; to return; } } - const started = await startRun(config, name, inputs); + const base = await headCommit(process.cwd()); + const started = await startRun(config, name, inputs, base ? { base, checkout: process.cwd() } : {}); if ('error' in started || options.driver !== 'local') { writeEnvelope(started); return; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 993010637..5b33d87d2 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -793,7 +793,7 @@ export async function startRun( config: LocalConfig, name: string, inputs: unknown = {}, - options: { driver?: DriverIdentity } = {}, + options: { driver?: DriverIdentity; base?: RunState['base']; checkout?: string } = {}, ): Promise { const cached = await loadCachedWorkflow(config, name); if (!cached) { @@ -817,6 +817,8 @@ export async function startRun( ...(Object.keys(tree.workflows).length > 0 ? { workflows: tree.workflows } : {}), steps: cached.definition.steps.map((step) => ({ id: step.id, status: 'pending' })), settledAttempts: [], + ...(options.base ? { base: options.base } : {}), + ...(options.checkout ? { checkout: options.checkout } : {}), createdAt: now, updatedAt: now, }; @@ -849,8 +851,8 @@ export interface NextOptions { running?: string[]; /** False to reconcile without issuing a new lease: `next` answers `done`, an unlisted lease, `hitl`, or `wait`. */ canTakeLease?: boolean; - /** The checkout the run's workspaces start from. The first `next` that sends it fixes the run's base. */ - base?: { ref: string; commit: string }; + /** The checkout the run's workspaces start from. It fixes the run's base only when `workflow start` recorded none. */ + base?: { ref?: string; commit: string }; /** Workspace modes this driver can run. Omitted means read-only, so a `new` or `continue` step waits. */ workspaceModes?: WorkspaceMode[]; /** Whether a person is present for HITL requests and interactive work on this next request. */ @@ -1103,10 +1105,10 @@ export function parseNextPayload(value: unknown): ParsedNextPayload | string { if (record.base !== undefined) { if (!record.base || typeof record.base !== 'object' || Array.isArray(record.base)) return 'base must be an object'; const base = record.base as Record; - if (typeof base.ref !== 'string' || base.ref === '' || typeof base.commit !== 'string' || base.commit === '') { - return 'base must include ref and commit'; + if ((base.ref !== undefined && (typeof base.ref !== 'string' || base.ref === '')) || typeof base.commit !== 'string' || base.commit === '') { + return 'base must include commit, and ref when it is set must not be empty'; } - request.base = { ref: base.ref, commit: base.commit }; + request.base = { ...(typeof base.ref === 'string' ? { ref: base.ref } : {}), commit: base.commit }; } const ignored = unreadFields(record, NEXT_PAYLOAD_FIELDS); const can = isRecord(record.driver) ? record.driver.can : undefined; diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index ff663a3a4..a962469c0 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -155,6 +155,8 @@ interface Lease { model?: string; defaultModel?: string; workspace?: WorkspaceContract; + /** The lease named a workspace this driver cannot read. The step fails instead of running in the active checkout. */ + unreadableWorkspace?: true; session?: { tool: string; id: string }; /** Set once the step has a directory to run in. */ cwd?: string; @@ -318,8 +320,8 @@ type ReportBody = Parameters[2]; function workspaceOf(value: unknown): WorkspaceContract | undefined { if (!isRecord(value) || (value.mode !== 'new' && value.mode !== 'continue') || typeof value.branch !== 'string') return undefined; - const base = isRecord(value.base) && typeof value.base.commit === 'string' && typeof value.base.ref === 'string' - ? { ref: value.base.ref, commit: value.base.commit } : undefined; + const base = isRecord(value.base) && typeof value.base.commit === 'string' && (value.base.ref === undefined || typeof value.base.ref === 'string') + ? { ...(typeof value.base.ref === 'string' ? { ref: value.base.ref } : {}), commit: value.base.commit } : undefined; const start = isRecord(value.start) && typeof value.start.commit === 'string' ? { commit: value.start.commit, ...(typeof value.start.branch === 'string' ? { branch: value.start.branch } : {}) } : undefined; if (!base || !start || typeof value.configCommit !== 'string' || !Array.isArray(value.merge)) return undefined; @@ -333,6 +335,7 @@ function parseLease(raw: unknown): Lease | undefined { const rawLease = raw as Record | undefined; if (!rawLease || typeof rawLease.step !== 'string' || typeof rawLease.attempt !== 'string' || typeof rawLease.revision !== 'number') return undefined; const env = stringMap(rawLease.env); + const workspace = workspaceOf(rawLease.workspace); const session = isRecord(rawLease.session) && typeof rawLease.session.tool === 'string' && typeof rawLease.session.id === 'string' ? { tool: rawLease.session.tool, id: rawLease.session.id } : undefined; const lease: Lease = { @@ -347,7 +350,7 @@ function parseLease(raw: unknown): Lease | undefined { ...(rawLease.interactive === true ? { interactive: true } : {}), ...(rawLease.readOnly === true ? { readOnly: true } : {}), ...(typeof rawLease.defaultModel === 'string' ? { defaultModel: rawLease.defaultModel } : {}), - ...(workspaceOf(rawLease.workspace) ? { workspace: workspaceOf(rawLease.workspace) } : {}), + ...(workspace ? { workspace } : rawLease.workspace !== undefined ? { unreadableWorkspace: true } : {}), ...(session ? { session } : {}), }; return (lease.command === undefined) === (lease.prompt === undefined) ? undefined : lease; @@ -359,6 +362,9 @@ function placeOf(lease: Lease, active: string): string { } async function stageLease(active: string, lease: Lease): Promise { + if (lease.unreadableWorkspace) { + return { code: 'workspace_base', message: `Step "${lease.step}" was leased a workspace this teamai cannot read. Update teamai.` }; + } if (!lease.workspace) { lease.cwd = active; lease.before = await snapshotWorktree(active); diff --git a/src/workflow/types.ts b/src/workflow/types.ts index f7767bb1f..d3ba579c9 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -234,8 +234,13 @@ export interface RunState { archive?: ArchivedPass[]; /** The round of a looping top-level workflow. */ round?: number; - /** The commit the run's workspaces and agent configuration start from. Set by the first `next` that sends `base`. */ - base?: { ref: string; commit: string }; + /** + * The commit the run's workspaces and agent configuration start from, and the branch it was on. Set by `workflow start` + * in a git checkout, else by the first `next` that sends `base`. A later `base` does not change it. + */ + base?: { ref?: string; commit: string }; + /** The checkout `workflow start` ran in. Local run state only: never on a lease or a record. */ + checkout?: string; /** Workspaces this run has handed out, by branch. A `continue` step stays with the driver that opened the copy. */ copies?: Record; createdAt: string; diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 23aff8cc5..a88156f28 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -22,7 +22,7 @@ export interface StepChanges { /** The git contract on a `new` or `continue` lease. Paths stay off it. */ export interface WorkspaceContract { mode: 'new' | 'continue'; - base: { ref: string; commit: string }; + base: { ref?: string; commit: string }; /** The commit agent configuration is restored from before the step runs. */ configCommit: string; branch: string; @@ -220,7 +220,7 @@ export interface WorkspaceRequest { driver: DriverIdentity; /** Modes this driver can run. Omitted means read-only only, so a workspace step waits. */ modes?: WorkspaceMode[]; - base?: { ref: string; commit: string }; + base?: { ref?: string; commit: string }; } /** diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 29c76b48f..702bac6fe 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -94,11 +94,14 @@ export function hooksPath(main: string): string { return path.join(main, '.teamai', 'workflow-hooks'); } -export async function headCommit(cwd: string): Promise<{ ref: string; commit: string } | undefined> { +/** The commit HEAD points to, and the branch when HEAD is not detached. */ +export async function headCommit(cwd: string): Promise<{ ref?: string; commit: string } | undefined> { const result = await git(cwd, ['rev-parse', 'HEAD']); if (result.code !== 0) return undefined; const commit = result.stdout.trim(); - return commit ? { ref: 'HEAD', commit } : undefined; + if (!commit) return undefined; + const ref = (await git(cwd, ['symbolic-ref', '--quiet', '--short', 'HEAD'])).stdout.trim(); + return ref ? { ref, commit } : { commit }; } /** Creates the worktree when it is missing. A branch that already exists is checked out as it is. */ From 05fabd37dc5cf1f7a605a9ee94872dc356b28b67 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:42:30 +0200 Subject: [PATCH 05/41] fix(workflow): observe a workspace prompt through the worktree's config --- src/__tests__/workflow-workspace.test.ts | 140 +++++++++++++++-------- src/workflow/local-driver.ts | 11 +- 2 files changed, 99 insertions(+), 52 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 54fde61ac..90157670d 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -22,6 +22,9 @@ import { readRun, writeRun } from '../workflow/store.js'; import type { DriverIdentity } from '../workflow/types.js'; import { branchName } from '../workflow/workspace.js'; import { pushIfOrigin, worktreePath } from '../workflow/worktree.js'; +import { launchHoldsCheckout } from '../launch-context.js'; +import { teamDelivery } from '../pull.js'; +import { pendingRecords } from '../workflow/records.js'; const dirs: string[] = []; @@ -56,6 +59,57 @@ function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }); } +/** A checkout whose team repo is its own `.teamai`, pulled once with Claude enabled, and a started run of `yaml`. */ +async function teamWorkspace(yaml: string): Promise<{ config: LocalConfig; workspace: string; runId: string }> { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-workspace-'))); + dirs.push(root); + const workspace = path.join(root, 'workspace'); + const knowledge = path.join(workspace, '.teamai'); + fs.mkdirSync(path.join(knowledge, 'skills', 'fe-skill'), { recursive: true }); + fs.mkdirSync(path.join(knowledge, 'workflows'), { recursive: true }); + fs.writeFileSync(path.join(knowledge, 'teamai.yaml'), [ + 'team: workspace', + 'mode: self', + 'repo: https://git.example.com/team/workspace.git', + 'provider: git', + '', + ].join('\n')); + fs.writeFileSync(path.join(knowledge, '.gitignore'), 'config.yaml\nstate.json\n'); + fs.writeFileSync(path.join(knowledge, 'skills', 'fe-skill', 'SKILL.md'), '---\nname: fe-skill\ndescription: fixture\n---\n\nTeam skill.\n'); + fs.writeFileSync(path.join(knowledge, 'workflows', 'build.yaml'), yaml); + fs.writeFileSync(path.join(workspace, 'tracked.txt'), 'clean\n'); + execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: workspace }); + execFileSync('git', ['add', '-A'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'base'], { cwd: workspace }); + const anchors = await resolveAnchors(workspace); + if (!anchors) throw new Error('the workspace has no git anchors'); + const partition = await resolvePartitionDir(anchors.projectAnchor); + fs.mkdirSync(partition, { recursive: true }); + fs.writeFileSync(path.join(partition, 'config.yaml'), [ + 'repo:', + ` localPath: ${knowledge}`, + ' remote: https://git.example.com/team/workspace.git', + ' kind: self', + ` businessRepoRoot: ${workspace}`, + 'username: tester', + 'updatePolicy: auto', + 'scope: project', + `projectRoot: ${workspace}`, + 'enabledAgents: [claude]', + '', + ].join('\n')); + fs.mkdirSync(path.join(workspace, '.claude')); + const { pull } = await import('../pull.js'); + await pull({ silent: true, cwd: workspace }); + const config = await detectProjectConfig(workspace); + if (!config) throw new Error('project config was not found'); + config.workflows = { tools: { default: 'claude' }, max_parallel: 1 }; + await syncWorkflowDefinitions(config); + const started = await startRun(config, 'build'); + if ('error' in started) throw new Error(started.error.message); + return { config, workspace, runId: String(started.data.run) }; +} + const driver = (workspace: string, machine = 'host'): DriverIdentity => ({ kind: 'local', machine, workspace }); afterEach(() => { @@ -636,22 +690,7 @@ steps: }); it('delivers a team skill into a new workspace and leaves it out of the branch', async () => { - const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-workspace-'))); - dirs.push(root); - const workspace = path.join(root, 'workspace'); - const knowledge = path.join(workspace, '.teamai'); - fs.mkdirSync(path.join(knowledge, 'skills', 'fe-skill'), { recursive: true }); - fs.mkdirSync(path.join(knowledge, 'workflows'), { recursive: true }); - fs.writeFileSync(path.join(knowledge, 'teamai.yaml'), [ - 'team: workspace', - 'mode: self', - 'repo: https://git.example.com/team/workspace.git', - 'provider: git', - '', - ].join('\n')); - fs.writeFileSync(path.join(knowledge, '.gitignore'), 'config.yaml\nstate.json\n'); - fs.writeFileSync(path.join(knowledge, 'skills', 'fe-skill', 'SKILL.md'), '---\nname: fe-skill\ndescription: fixture\n---\n\nTeam skill.\n'); - fs.writeFileSync(path.join(knowledge, 'workflows', 'build.yaml'), `schemaVersion: v1alpha1 + const { config, workspace, runId } = await teamWorkspace(`schemaVersion: v1alpha1 name: build description: Read the delivered team skill. steps: @@ -659,36 +698,6 @@ steps: workspace: new prompt: Read the team skill. `); - fs.writeFileSync(path.join(workspace, 'tracked.txt'), 'clean\n'); - execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: workspace }); - execFileSync('git', ['add', '-A'], { cwd: workspace }); - execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'base'], { cwd: workspace }); - const anchors = await resolveAnchors(workspace); - if (!anchors) throw new Error('the workspace has no git anchors'); - const partition = await resolvePartitionDir(anchors.projectAnchor); - fs.mkdirSync(partition, { recursive: true }); - fs.writeFileSync(path.join(partition, 'config.yaml'), [ - 'repo:', - ` localPath: ${knowledge}`, - ' remote: https://git.example.com/team/workspace.git', - ' kind: self', - ` businessRepoRoot: ${workspace}`, - 'username: tester', - 'updatePolicy: auto', - 'scope: project', - `projectRoot: ${workspace}`, - 'enabledAgents: [claude]', - '', - ].join('\n')); - fs.mkdirSync(path.join(workspace, '.claude')); - const { pull } = await import('../pull.js'); - await pull({ silent: true, cwd: workspace }); - const config = await detectProjectConfig(workspace); - if (!config) throw new Error('project config was not found'); - config.workflows = { tools: { default: 'claude' }, max_parallel: 1 }; - await syncWorkflowDefinitions(config); - const started = await startRun(config, 'build'); - if ('error' in started) throw new Error(started.error.message); const adapter: LocalToolAdapter = { async runInteractive() { return { started: false, exitCode: 1, failure: { code: 'tool_failed', message: 'interactive is not used' } }; @@ -700,10 +709,10 @@ steps: }, }; - const done = await runLocalDriver(config, String(started.data.run), workspace, { adapter }); + const done = await runLocalDriver(config, runId, workspace, { adapter }); expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - const run = await readRun(config, String(started.data.run)); + const run = await readRun(config, runId); const ask = typeof run === 'object' ? run.steps.find((step) => step.id === 'ask') : undefined; expect(ask && 'outputs' in ask ? ask.outputs : undefined).toMatchObject({ skill: expect.stringContaining('Team skill.') }); const branch = ask && 'changes' in ask ? ask.changes?.branch : undefined; @@ -721,6 +730,41 @@ steps: expect(delivered?.rev).toBe(member?.rev); }); + it('observes a prompt in a new workspace through the worktree, so its tool keeps that delivery and the record its revision', async () => { + const { config, workspace, runId } = await teamWorkspace(`schemaVersion: v1alpha1 +name: build +description: Review in a workspace. +steps: + - id: ask + workspace: new + prompt: Review the change. +`); + // A member's own edit to a delivered skill is not a local change of the worktree the step runs in. + fs.appendFileSync(path.join(workspace, '.claude', 'skills', 'fe-skill', 'SKILL.md'), 'Member edit.\n'); + let holds: boolean | undefined; + let revision: string | undefined; + const adapter: LocalToolAdapter = { + async runInteractive() { + return { started: false, exitCode: 1, failure: { code: 'tool_failed', message: 'interactive is not used' } }; + }, + async run(_tool, input) { + const own = await detectProjectConfig(input.cwd); + holds = own && input.launch ? await launchHoldsCheckout(own, input.launch) : false; + const delivery = own ? await teamDelivery(own, 'claude') : undefined; + revision = delivery?.complete ? delivery.commit : undefined; + return { started: true, exitCode: 0 }; + }, + }; + + const done = await runLocalDriver(config, runId, workspace, { adapter }); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(holds).toBe(true); + expect(revision).toMatch(/^[0-9a-f]{40}$/); + const record = (await pendingRecords(config)).find((item) => item.run === runId && item.phase === 'end'); + expect(record?.steps?.[0]).toMatchObject({ agent: { team: { revision, localChanges: false } } }); + }); + it('keeps hooksPath on the workflow worktree and runs from a linked checkout', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index a962469c0..30af9df75 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -14,6 +14,7 @@ import { type ToolName, } from '../resources/agent-format.js'; import { observeLaunch } from '../agent-observation.js'; +import { detectProjectConfig } from '../config.js'; import { acquireLock, releaseLock } from '../update.js'; import { commandSignalPolicy, exitLike, terminalReachesCommand } from '../env-exec.js'; import { workflowLaunch } from '../launch-context.js'; @@ -581,7 +582,9 @@ async function promptReport( if (!supportedTool(selected)) { return giveUp({ code: 'tool_unsupported', message: `The local workflow tool "${selected}" is not supported` }, selected); } - const loaded = lease.agent ? await loadAgent(selected, lease.agent, config, options.toolPaths) : undefined; + // A worktree has its own pull record, so the tool launched there is observed and bound as that checkout. + const stepConfig = lease.workspace && lease.cwd ? await detectProjectConfig(lease.cwd) ?? config : config; + const loaded = lease.agent ? await loadAgent(selected, lease.agent, stepConfig, options.toolPaths) : undefined; const agent = loaded?.spec; if (lease.agent && !agent) { return giveUp({ @@ -590,7 +593,7 @@ async function promptReport( }, selected); } const chosen = await chooseStepModel({ - step: lease.step, tool: selected, model: lease.model, defaultModel: lease.defaultModel, agent: agent ?? undefined, config, + step: lease.step, tool: selected, model: lease.model, defaultModel: lease.defaultModel, agent: agent ?? undefined, config: stepConfig, }); if (!chosen.ok) return giveUp(chosen.failure, selected); if (agent && localToolFamily(selected) === 'codex') { @@ -607,7 +610,7 @@ async function promptReport( : {}; const [observed, launch] = await Promise.all([ observeLaunch({ - config, + config: stepConfig, tool: selected, cwd, ...(lease.agent && loaded ? { definition: { stem: lease.agent, file: loaded.file, ...definitionModel } } : {}), @@ -615,7 +618,7 @@ async function promptReport( ? { chosen: { model: chosen.model.name, ...(chosen.model.effort ? { effort: chosen.model.effort } : {}), ...(chosen.alias ? { alias: chosen.alias } : {}) } } : {}), }), - workflowLaunch(config, selected), + workflowLaunch(stepConfig, selected), ]); let result: LocalToolResult; try { From c01ae7085158727f7a960efd7e5582e1cd45802a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 01:56:41 +0200 Subject: [PATCH 06/41] fix(workflow): stage only agent paths that exist restoreUneditedConfig staged AGENT_PATHS with one git add. Git exits 128 when any listed path matches no file in the worktree or the index, and then stages nothing, so a copy kept the base agent file in its index after a step that edited none. --- src/__tests__/workflow-workspace.test.ts | 30 ++++++++++++++++++++++++ src/workflow/worktree.ts | 14 ++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 90157670d..584f780c6 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -1395,6 +1395,36 @@ steps: expect(git(['show', `${branch}:AGENTS.md`], workspace)).toBe('edited\n'); }); + it('leaves the copy index clean when a prompt stops before it runs on a branch without most tool directories', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Putting the branch agent file back also puts its index entry back. +steps: + - id: work + needs: [] + workspace: new + workflow: + steps: + - id: edit + needs: [] + workspace: continue + command: node -e "require('fs').writeFileSync('AGENTS.md', 'edited\\n')" + - id: ask + needs: [edit] + workspace: continue + prompt: Look +`); + fs.writeFileSync(path.join(workspace, 'AGENTS.md'), 'base\n'); + execFileSync('git', ['add', 'AGENTS.md'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'agents'], { cwd: workspace }); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const copy = worktreePath(workspace, branchName(runId, 'work', 1)); + expect(git(['status', '--porcelain'], copy)).toBe(''); + }); + it('does not publish conflict markers when a prompt fails', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 702bac6fe..68f111a09 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -238,10 +238,22 @@ export async function restoreUneditedConfig(dir: string, _commit: string, saved: await fs.rm(saved, { recursive: true, force: true }); // `checkout` of the base copies staged those paths. Put the index back with the files, or a step that // edits nothing still looks dirty and the empty commit fails it. - await git(dir, ['add', '-A', '--', ...AGENT_PATHS]); + const present = await presentAgentPaths(dir); + if (present.length > 0) await git(dir, ['add', '-A', '--', ...present]); if (unresolved.length > 0) await restoreConflictStages(dir, conflicts, unresolved); } +/** Agent paths in the worktree or the index. `git add` stages nothing when one of its paths matches neither. */ +async function presentAgentPaths(dir: string): Promise { + const indexed = await gitPaths(dir, ['ls-files', '--', ...AGENT_PATHS]); + const present: string[] = []; + for (const relative of AGENT_PATHS) { + const tracked = indexed.some((file) => file === relative || file.startsWith(`${relative}/`)); + if (tracked || await fs.lstat(path.join(dir, relative)).then(() => true, () => false)) present.push(relative); + } + return present; +} + async function gitPaths(dir: string, args: string[]): Promise { const listed = await git(dir, [...args.slice(0, 1), '-z', ...args.slice(1)]); return listed.stdout.split('\0').filter((file) => file !== ''); From 37e348cd92bbb255eb67712d648fc614cb2c7efc Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 02:03:31 +0200 Subject: [PATCH 07/41] fix(workflow): lease a ready step past one this driver holds back next answered wait at the first ready step whose workspace the driver could not take, so a driver without new never got a ready read-only step. Settling now returns every ready leaf in order, next leases the first this driver can take, and it waits only when every ready step is held back, naming each one. --- docs/dev/workflow-engine.md | 4 +- docs/guide/workflow-advanced.md | 4 +- docs/guide/zh-CN/workflow-advanced.md | 4 +- src/__tests__/workflow-engine.test.ts | 64 ++++++++++++++++++++ src/workflow/engine.ts | 84 +++++++++++++++++---------- 5 files changed, 122 insertions(+), 38 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 4cb8a35cc..088db3a28 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -7,11 +7,11 @@ Each rule below spans more than one file. The rule names the code that owns it, ## Rules that span files - **A run ends only in `settleRun`.** The shared `finishRun` path calls it when settling reaches the top workflow's result, before it hands any lease back or out, and on a call that takes no lease too. `reportAttempt` first reconciles earlier reports, so an end cancels a later report before it can advance the revision. It records an accepted outcome and never ends the run. After a failed report, the next `next` skips the dependents and ends the run. Respond, cancel and retry in `run-controls.ts`, and next in `engine.ts`, share that path. Whole-run cancel settles directly as `cancelled`, without reconciling another round. To run code when a run ends, hook into `finishRun`. Do network I/O only after `withRun` releases the run lock. That path queues the end record, a local write, under the lock, and saves whether it did in the run's `endRecordQueued` field. When `writeRun` then fails, the queue is one save ahead of the run file, and the next `next` settles the run again and overwrites the queued `.end.json`. A later `next` reports a record state only for a run whose field is true (`runRecordState` in `records.ts`). -- **Reconcile the latest report before applying another or handing out a lease.** `reconcileRun`, shared by `leaseNext` and `reportAttempt`, settles every driverless transition to a fixpoint at the run's current revision, through `settleScope` and `settleItems`, including workflow starts, fan-out expansion, render failures and bound exhaustion. `reportAttempt` advances the revision only after it, so only the latest revision can be unreconciled. A reached end applies before a loop decision. Only after reconciliation does `leaseNext` hand back an omitted open lease or issue the first ready leaf in declaration order. A running workflow occupies its parent's position in that order. `canTakeLease: false` still reconciles driverless transitions. Every changed run is saved, including a `wait`. +- **Reconcile the latest report before applying another or handing out a lease.** `reconcileRun`, shared by `leaseNext` and `reportAttempt`, settles every driverless transition to a fixpoint at the run's current revision, through `settleScope` and `settleItems`, including workflow starts, fan-out expansion, render failures and bound exhaustion. `reportAttempt` advances the revision only after it, so only the latest revision can be unreconciled. A reached end applies before a loop decision. Only after reconciliation does `leaseNext` hand back an omitted open lease or issue the first ready leaf in declaration order that this driver can take. A running workflow occupies its parent's position in that order. Settling returns every ready leaf in that order (`ready`), and `leaseNext` walks it. A ready item does not count against `max_parallel` until it is leased, and one `next` leases at most one step, so a held-back item leaves its room to the next one without exceeding the cap. `canTakeLease: false` still reconciles driverless transitions. Every changed run is saved, including a `wait`. - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It waits, and does not hand the lease out, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It holds the step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow-advanced.md b/docs/guide/workflow-advanced.md index df5ec3606..bc5d321c4 100644 --- a/docs/guide/workflow-advanced.md +++ b/docs/guide/workflow-advanced.md @@ -35,8 +35,8 @@ Each call to `next` does this, in order: 1. It settles the run. If the run has ended, it answers `done`. 2. It hands back an open lease of this driver that `running` leaves out. A restarted driver sends `"running":[]` and gets its open leases back, one per call. -3. It leases the first ready step in file order. The steps of a sub-workflow take the place of the step that runs it. -4. When every ready step is leased, it answers `wait`, and `reason` names the first open lease. +3. It leases the first ready step in file order. The steps of a sub-workflow take the place of the step that runs it. A step that needs a workspace mode `driver.can.workspace` does not list, or a workspace another driver opened, is held back, and the next ready step is leased. +4. When every ready step is leased, it answers `wait`, and `reason` names the first open lease. When every ready step is held back, `reason` names each one, separated by `; `. A driver that sends no `running` gets its open lease back on every call, so it runs one step at a time. diff --git a/docs/guide/zh-CN/workflow-advanced.md b/docs/guide/zh-CN/workflow-advanced.md index b01f9eb81..959bbcb9a 100644 --- a/docs/guide/zh-CN/workflow-advanced.md +++ b/docs/guide/zh-CN/workflow-advanced.md @@ -35,8 +35,8 @@ steps: 1. 结算运行。如果运行已经结束，返回 `done`。 2. 把本驱动的、`running` 里没有列出的未结租约交还给它。重启后的驱动发送 `"running":[]`，每次调用拿回一个未结租约。 -3. 按文件顺序租出第一个就绪的步骤。子工作流的步骤位于运行它的那个步骤的位置。 -4. 每个就绪的步骤都已租出时，返回 `wait`，`reason` 指出第一个未结租约。 +3. 按文件顺序租出第一个就绪的步骤。子工作流的步骤位于运行它的那个步骤的位置。需要 `driver.can.workspace` 未列出的工作区模式的步骤，或使用其他驱动打开的工作区的步骤，会被暂缓，改为租出下一个就绪的步骤。 +4. 每个就绪的步骤都已租出时，返回 `wait`，`reason` 指出第一个未结租约。每个就绪的步骤都被暂缓时，`reason` 逐一列出它们，以 `; ` 分隔。 不发送 `running` 的驱动每次调用都会拿回自己的未结租约，因此一次只运行一个步骤。 diff --git a/src/__tests__/workflow-engine.test.ts b/src/__tests__/workflow-engine.test.ts index 2905ef419..7855bc9d3 100644 --- a/src/__tests__/workflow-engine.test.ts +++ b/src/__tests__/workflow-engine.test.ts @@ -2501,6 +2501,70 @@ describe('parallel leases', () => { }); }); +describe('held-back steps', () => { + const base = { commit: 'b'.repeat(40) }; + const allModes = ['read-only', 'new', 'continue'] as const; + + it('leases a ready step this driver can run past one it holds back, and keeps that one pending', async () => { + const { config, runId } = await startBuild(graph(` - id: a + workspace: new + command: node a.js + - id: b + needs: [] + command: node b.js +`)); + + const next = await leaseNext(config, runId, driverA, { workspaceModes: ['read-only'] }); + + expect(next).toMatchObject({ data: { lease: { step: 'b' } } }); + expect(await readRun(config, runId)).toMatchObject({ steps: [{ id: 'a', status: 'pending' }, { id: 'b', status: 'leased' }] }); + await reportAttempt(config, runId, { ...leaseIn(next), outcome: 'success', driver: driverA }); + expect(await leaseNext(config, runId, driverA, { workspaceModes: ['read-only'] })).toEqual({ data: { + wait: { reason: 'step "a" needs workspace new, which this driver does not provide' }, cancelled: [], + } }); + }); + + it('waits when every ready step is held back, and names each one in order', async () => { + const { config, runId } = await startBuild(graph(` - id: a + workspace: new + command: node a.js + - id: c + needs: [] + workspace: new + command: node c.js +`)); + + const next = await leaseNext(config, runId, driverA, { workspaceModes: ['read-only'] }); + + expect(next).toEqual({ data: { wait: { + reason: 'step "a" needs workspace new, which this driver does not provide; step "c" needs workspace new, which this driver does not provide', + }, cancelled: [] } }); + }); + + it('leases the next item when the first one is held back, and still starts at most max_parallel items', async () => { + const { config, runId } = await startTree({ build: workflow('build', ` - id: build + for_each: { items: inputs.targets, max_parallel: 1 } + workspace: new + command: node build.js +`, 'inputs:\n targets: [string]\n') }, 'build', { targets: ['web', 'api', 'cli'] }); + const web = leaseIn(await leaseNext(config, runId, driverB, { workspaceModes: [...allModes], base })); + await reportAttempt(config, runId, { ...web, outcome: 'released', driver: driverB }); + + const api = await leaseNext(config, runId, driverA, { workspaceModes: [...allModes], base }); + const full = await leaseNext(config, runId, driverA, { workspaceModes: [...allModes], base, running: [leaseIn(api).attempt] }); + + expect(web.step).toBe('build[0]'); + expect(api).toMatchObject({ data: { lease: { step: 'build[1]' } } }); + expect(full).toEqual({ data: { wait: { reason: 'build[1] is leased' }, cancelled: [] } }); + expect(await readRun(config, runId)).toMatchObject({ steps: [ + { id: 'build', status: 'running' }, + { id: 'build[0]', status: 'pending' }, + { id: 'build[1]', status: 'leased' }, + { id: 'build[2]', status: 'pending' }, + ] }); + }); +}); + describe('causal settlement order', () => { it.each(['workflow', 'for_each'])('checks an end-only condition after a %s step settles', async (action) => { const build = action === 'workflow' diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 5b33d87d2..eb922365c 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -445,12 +445,19 @@ function skipUnreachable(run: RunState, scope: Scope, states: StepState[], { con return ready; } +/** A leaf step that can be leased, with what its lease renders to. */ +interface Ready { + lease: StepState; + definition: WorkflowStep; + rendered: RenderedLease; +} + /** - * How settling a workflow of the run ended: a step to lease, the workflow's outcome, nothing to do until an open - * lease is reported, or a run file that cannot settle. + * How settling a workflow of the run ended: the steps to lease, in the order they were found, the workflow's outcome, + * nothing to do until an open lease is reported, or a run file that cannot settle. */ type Settled = - | { lease: StepState; definition: WorkflowStep; rendered: RenderedLease } + | { ready: Ready[] } | WorkflowResult | { inProgress: true } | { unreadable: string }; @@ -570,7 +577,7 @@ function advanceInstance( } if (definition.workflow === undefined) { const rendered = renderLease(scope, states, state, definition); - if (!('code' in rendered)) return { lease: state, definition, rendered }; + if (!('code' in rendered)) return { ready: [{ lease: state, definition, rendered }] }; failStep(state, rendered, run.revision); return undefined; } @@ -582,8 +589,9 @@ function advanceInstance( } /** - * Settles items at the current revision and returns the first leaf available to lease. An item waits for its - * `after` dependencies and for room under `max_parallel`. + * Settles items at the current revision and returns the leaves available to lease. An item waits for its + * `after` dependencies and for room under `max_parallel`. A ready leaf does not take room until it is leased, and + * `next` leases one step per call, so a held-back item leaves its room to the next one. */ function settleItems( run: RunState, scope: Scope, states: StepState[], state: StepState, definition: WorkflowStep, @@ -599,7 +607,7 @@ function settleItems( const maxParallel = definition.forEach?.maxParallel; let started = startedItems(items); let changed = false; - let candidate: Settled | undefined; + const ready: Ready[] = []; for (const [index, item] of items.entries()) { if (item.status === 'pending' && standing(index) === 'skip') { // A skip makes the items after it skippable wherever they are in item order, so it reaches them all now. @@ -615,7 +623,7 @@ function settleItems( if (item.status !== 'running' && (item.status !== 'pending' || standing(index) !== 'ready' || (maxParallel !== undefined && started >= maxParallel))) continue; const advanced = advanceInstance(run, scope, states, item, definition); if (advanced && 'unreadable' in advanced) return advanced; - if (advanced && 'lease' in advanced) candidate ??= advanced; + if (advanced && 'ready' in advanced) ready.push(...advanced.ready); if (advanced) continue; changed = true; // Starting an item changes no other item's standing, so the scan goes on. Settling one can make an earlier item @@ -625,7 +633,7 @@ function settleItems( } if (changed) continue; if (items.every(settled)) return itemsSettled(items, run.revision, run); - return candidate ?? { inProgress: true }; + return ready.length > 0 ? { ready } : { inProgress: true }; } } @@ -637,8 +645,8 @@ function workflowOutputs(scope: Scope, states: StepState[], paths: Map defined.get(localId(scope, state)); for (;;) { let changed = false; - let candidate: Settled | undefined; + const ready: Ready[] = []; const canRun = skipUnreachable(run, scope, states, reads); for (const state of states) { if (state.status !== 'running' && (state.status !== 'pending' || !canRun(state))) continue; @@ -682,7 +690,7 @@ function settleScope(run: RunState, scope: Scope): Settled { } else { const advanced = advanceInstance(run, scope, states, state, definition, reads.items.get(localId(scope, state))); if (advanced && 'unreadable' in advanced) return advanced; - if (advanced && 'lease' in advanced) candidate ??= advanced; + if (advanced && 'ready' in advanced) ready.push(...advanced.ready); if (advanced) continue; } changed = true; @@ -727,7 +735,7 @@ function settleScope(run: RunState, scope: Scope): Settled { if (decided === 'again') continue; return decided === 'ended' ? result : { ...result, outcome: 'failed', failure: decided }; } - if (candidate) return candidate; + if (ready.length > 0) return { ready }; if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running')) return { inProgress: true }; return { unreadable: `Run ${run.id} has pending steps whose needs cannot be met` }; } @@ -914,25 +922,37 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv const hitl = options.hitl ? waitingHitl(run) : undefined; return hitl ? answer({ hitl }) : wait(); } - if (result.definition.interactive && !options.hitl) { - const failure = interactiveHitlRequired(run.id, result.lease.id); - failStep(result.lease, failure, run.revision); - continue; - } - const located = locateStep(run, result.lease.id); - if (!located) return { save: false, envelope: error('run_unreadable', `Run ${run.id} has no definition for step "${result.lease.id}"`) }; - const plan = planWorkspace(run, located.scope, result.lease, result.definition, { - driver, modes: options.workspaceModes, base: run.base ?? options.base, - }); - if (plan.defer) return answer({ wait: { reason: plan.defer } }); - if (plan.fail) { - failStep(result.lease, plan.fail, run.revision); - continue; + // A step this driver must not take is held back, and the next ready one is tried. A failed step can change + // what is ready, so the run settles again. + const held: string[] = []; + let failed = false; + for (const { lease: state, definition, rendered } of result.ready) { + if (definition.interactive && !options.hitl) { + failStep(state, interactiveHitlRequired(run.id, state.id), run.revision); + failed = true; + break; + } + const located = locateStep(run, state.id); + if (!located) return { save: false, envelope: error('run_unreadable', `Run ${run.id} has no definition for step "${state.id}"`) }; + const plan = planWorkspace(run, located.scope, state, definition, { + driver, modes: options.workspaceModes, base: run.base ?? options.base, + }); + if (plan.defer) { + held.push(plan.defer); + continue; + } + if (plan.fail) { + failStep(state, plan.fail, run.revision); + failed = true; + break; + } + if (plan.contract && options.base && !run.base) run.base = options.base; + const step = lease(state, { attempt: `a${randomBytes(4).toString('hex')}`, revision: run.revision, driver }); + rememberCopy(run, plan, driver); + return answer(leaseData(run, step, definition, rendered, plan)); } - if (plan.contract && options.base && !run.base) run.base = options.base; - const step = lease(result.lease, { attempt: `a${randomBytes(4).toString('hex')}`, revision: run.revision, driver }); - rememberCopy(run, plan, driver); - return answer(leaseData(run, step, result.definition, result.rendered, plan)); + if (failed) continue; + return answer({ wait: { reason: held.join('; ') } }); } const located = locateStep(run, leased.id); if (!located) return { save: false, envelope: error('run_unreadable', `Run ${run.id} has no definition for step "${leased.id}"`) }; From 0bfa3dbac5f0465d0491ba9f8c9d742a16a724cd Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 02:15:54 +0200 Subject: [PATCH 08/41] fix(workflow): run a read-only step inside a copy in that copy A command such as `npm test` after an implement step inside a `new` copy ran in the member checkout, so it tested code without the change. A read-only step under a copy owner now leases a read-only contract on the branch its need wrote (or the owner's branch), runs in that worktree on the base agent files, commits nothing, and reports the commit it read as `read`. A copy the reader changed is reset to that commit, so a later step there does not commit the change. Needs that wrote two branches fail with workspace_base. Part of #1246 (F4). --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 8 +- docs/guide/zh-CN/workflow.md | 8 +- src/__tests__/workflow-workspace.test.ts | 338 ++++++++++++++++++++++- src/workflow/engine.ts | 57 ++-- src/workflow/local-driver.ts | 65 ++++- src/workflow/step-state.ts | 9 + src/workflow/workspace.ts | 73 ++++- 8 files changed, 512 insertions(+), 48 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 088db3a28..c29c5dbea 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease only for `new` and `continue`. It holds the step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy, it runs `git reset --hard` to the commit it read and `git clean -fd`, so a later writer never commits a reader's edits. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index edc713af7..f227ee11d 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -121,7 +121,7 @@ An agent-specific entry wins over `default`. `claude`, `codex`, and `opencode` a teamai workflow start build --driver local ``` -`--driver local` runs a step with no `workspace` in the current checkout. A command there is read-only: if it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails and leaves that change in place. This includes a change inside an existing untracked directory; leaving that directory unchanged does not fail the step. A step with `workspace: new` changes files on its own branch. See [Workspaces](#workspaces). Steps that share the current checkout run one at a time. Steps in different worktrees run together up to [`workflows.max_parallel`](workflow-advanced.md#run-local-commands-in-parallel). Each attempt uses `result.json` under the scope's TeamAI data directory, outside the checkout. The driver calls the same `next` and `report` operations shown below: +`--driver local` runs a step with no `workspace` in the current checkout, unless an enclosing step opened a workspace. A command there is read-only: if it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails and leaves that change in place. This includes a change inside an existing untracked directory; leaving that directory unchanged does not fail the step. A step with `workspace: new` changes files on its own branch. See [Workspaces](#workspaces). Steps that share the current checkout run one at a time. Steps in different worktrees run together up to [`workflows.max_parallel`](workflow-advanced.md#run-local-commands-in-parallel). Each attempt uses `result.json` under the scope's TeamAI data directory, outside the checkout. The driver calls the same `next` and `report` operations shown below: ```text teamai workflow start build @@ -129,7 +129,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. +`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. The run is stored in this scope's TeamAI data directory. The command exits when it finishes. A local run that ends `failed` or `cancelled` exits with status 1. No workflow process keeps running. @@ -137,7 +137,9 @@ Use `workflow respond ` for a waiting HITL request, `workfl ## Workspaces -A command with no `workspace` is read-only, even inside a workspace a parent opened. It runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. +A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. + +Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it, the worktree is reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. `workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 95b44d187..8749e0604 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -121,7 +121,7 @@ workflows: teamai workflow start build --driver local ``` -`--driver local` 在当前检出目录里运行没有 `workspace` 的步骤。这里的命令是只读的：退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤失败，改动留在原地。已有未跟踪目录里的文件变化也算；该目录保持不变时，步骤不会因此失败。`workspace: new` 的步骤在自己的分支上改文件。见[工作区](#工作区)。共用当前检出目录的步骤一次只运行一个。不同工作树里的步骤可以一起运行，上限是 [`workflows.max_parallel`](workflow-advanced.md#并行运行本地命令)。每次 attempt 的 `result.json` 位于当前 scope 的 TeamAI 数据目录中，不在产品检出目录内。驱动调用的 `next` 和 `report` 与下面相同： +`--driver local` 在当前检出目录里运行没有 `workspace` 的步骤，除非外层步骤打开了工作区。这里的命令是只读的：退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤失败，改动留在原地。已有未跟踪目录里的文件变化也算；该目录保持不变时，步骤不会因此失败。`workspace: new` 的步骤在自己的分支上改文件。见[工作区](#工作区)。共用当前检出目录的步骤一次只运行一个。不同工作树里的步骤可以一起运行，上限是 [`workflows.max_parallel`](workflow-advanced.md#并行运行本地命令)。每次 attempt 的 `result.json` 位于当前 scope 的 TeamAI 数据目录中，不在产品检出目录内。驱动调用的 `next` 和 `report` 与下面相同： ```text teamai workflow start build @@ -129,7 +129,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 +`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 运行状态保存在当前 scope 的 TeamAI 数据目录里。命令结束后进程就退出。本地运行以 `failed` 或 `cancelled` 结束时，退出码是 1。不会留下常驻的工作流进程。 @@ -137,7 +137,9 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom ## 工作区 -没有 `workspace` 的命令是只读的，即使它位于父步骤打开的工作区里。它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 +没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 + +在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区时，worktree 重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 `workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 584f780c6..386979f2c 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -3,6 +3,7 @@ * branches, a later step merges them, and `continue` stays with the driver that opened it. */ import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -13,7 +14,7 @@ import { resolvePartitionDir } from '../utils/partition.js'; import type { LocalConfig } from '../types.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; import * as engine from '../workflow/engine.js'; -import { leaseNext, reportAttempt, startRun } from '../workflow/engine.js'; +import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; import { runLocalDriver } from '../workflow/local-driver.js'; import { cancelRun, retryStep } from '../workflow/run-controls.js'; import type { LocalToolAdapter } from '../workflow/local-tool-adapter.js'; @@ -2097,3 +2098,338 @@ setInterval(() => {}, 1000); expect(git(['show', `${branchName(runId, 'work', 1)}:AGENTS.md`], workspace)).toBe('edited\n'); }); }); + +describe('read-only steps in a workspace', () => { + const EXAMPLE = `schemaVersion: v1alpha1 +name: build +description: A test reads the change it needs. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + prompt: Implement the change. + - id: test + needs: [implement] + command: node verify.js +`; + const modes = ['read-only', 'new', 'continue'] as const; + const commit = (letter: string): string => letter.repeat(40); + + interface Leased { step: string; attempt: string; revision: number; workspace?: { mode: string; branch: string; base: { commit: string } } } + + function leaseOf(envelope: Awaited>): Leased { + if ('error' in envelope || !envelope.data.lease) throw new Error(`expected a lease, got ${JSON.stringify(envelope)}`); + return envelope.data.lease as Leased; + } + + /** Leases and reports `change.implement` of the example as `owner`, with a commit `to` on the copy's branch. */ + async function implemented(yaml = EXAMPLE): Promise<{ config: LocalConfig; workspace: string; runId: string; owner: DriverIdentity; base: { commit: string }; branch: string }> { + const { config, workspace, runId } = await fixture(yaml); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const implement = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + const branch = branchName(runId, 'change', 1); + expect(implement).toMatchObject({ step: 'change.implement', workspace: { mode: 'continue', branch } }); + const reported = await reportAttempt(config, runId, { + attempt: implement.attempt, revision: implement.revision, outcome: 'success', driver: owner, + changes: { branch, base: base.commit, from: base.commit, to: commit('c'), commits: [commit('c')] }, + }); + expect(reported).toMatchObject({ data: { accepted: true } }); + return { config, workspace, runId, owner, base, branch }; + } + + it('leases a read-only step inside a copy on the branch its need wrote, merging nothing', async () => { + const { config, runId, owner, base, branch } = await implemented(); + + const test = await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] }); + + expect(test).toMatchObject({ data: { lease: { step: 'change.test', workspace: { + mode: 'read-only', base: { commit: base.commit }, configCommit: base.commit, branch, start: { commit: commit('c') }, merge: [], + } } } }); + }); + + it('holds a reader in a copy back from a driver without continue, and from a driver that does not own the copy', async () => { + const { config, runId, owner, base, branch } = await implemented(); + + const reader = await leaseNext(config, runId, owner, { base, workspaceModes: ['read-only', 'new'] }); + const other = await leaseNext(config, runId, driver(owner.workspace, 'other'), { base, workspaceModes: [...modes] }); + + expect(reader).toEqual({ data: { wait: { reason: `step "change.test" reads workspace ${branch}, which needs continue` }, cancelled: [] } }); + expect(other).toEqual({ data: { wait: { reason: 'step "change.test" reads a workspace owned by another driver' }, cancelled: [] } }); + }); + + it('runs an item reader on its own item branch, after the items its item waits for, and publishes only writer branches', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Each item reads its own change. +inputs: + items: [{ id: string, after: [string] }] +steps: + - id: fan + workspace: new + for_each: + items: inputs.items + workflow: + steps: + - id: implement + prompt: Implement {{ item.id }}. + - id: fix + needs: [implement] + workspace: new + command: node fix.js + - id: test + needs: [implement] + command: node verify.js + - id: review + needs: [test] + command: node review.js +`, { items: [{ id: 'a', after: [] }, { id: 'b', after: ['a'] }] }); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const ran = new Map(); + for (;;) { + const next = await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] }); + if ('data' in next && next.data.done) break; + const lease = leaseOf(next); + const contract = lease.workspace; + ran.set(lease.step, contract?.branch); + const to = createHash('sha1').update(lease.step).digest('hex'); + await reportAttempt(config, runId, { + attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver: owner, + ...(contract?.mode === 'read-only' ? { read: { branch: contract.branch, base: base.commit, commit: base.commit } } : {}), + ...(contract && contract.mode !== 'read-only' ? { changes: { branch: contract.branch, base: base.commit, from: base.commit, to, commits: [to] } } : {}), + }); + } + + for (const item of ['fan[0]', 'fan[1]']) { + expect(ran.get(`${item}.test`), item).toBe(branchName(runId, `${item}.implement`, 1)); + expect(ran.get(`${item}.review`), item).toBe(branchName(runId, `${item}.implement`, 1)); + } + const run = await readRun(config, runId); + const fan = typeof run === 'object' ? run.steps.find((step) => step.id === 'fan') : undefined; + expect(fan && 'results' in fan ? fan.results?.map((item) => item.branch) : fan).toEqual([ + branchName(runId, 'fan[0].fix', 1), + branchName(runId, 'fan[1].fix', 1), + ]); + }); + + it('reads read from a report payload, and refuses one without branch, base, and commit', () => { + const report = { attempt: 'a00000000', revision: 1, outcome: 'success', driver: driver('/work') }; + const read = { branch: 'teamai/r/change/p1', base: commit('b'), commit: commit('c') }; + + const parsed = parseReport({ ...report, read }); + + expect(parsed).toEqual({ report: { ...report, read }, ignored: [] }); + expect(parseReport({ ...report, read: { branch: read.branch, base: read.base } })).toBe('read must include branch, base, and commit'); + }); + + it('accepts read for a read-only lease, refuses changes or a success without read, and keeps the copy for the reader', async () => { + const { config, runId, owner, base, branch } = await implemented(); + const test = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + const read = { branch, base: base.commit, commit: commit('d') }; + const report = (fields: Partial) => reportAttempt(config, runId, { + attempt: test.attempt, revision: test.revision, outcome: 'success', driver: owner, ...fields, + }); + + expect(await report({ changes: { branch, base: base.commit, from: commit('c'), to: commit('c'), commits: [] } })) + .toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining('leased read-only') } }); + expect(await report({})).toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining('no read') } }); + expect(await report({ read: { ...read, branch: 'teamai/elsewhere' } })) + .toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining('teamai/elsewhere') } }); + const accepted = await report({ read }); + + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'change.test', status: 'succeeded' }, cleanup: true } }); + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ read }); + expect(run.copies?.[branch]).toEqual({ owner, commit: commit('c') }); + }); + + it('keeps the copy when a writer reports while its reader is pending, and refuses read for a writable lease', async () => { + const { config, workspace, runId } = await fixture(EXAMPLE); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const implement = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + const branch = branchName(runId, 'change', 1); + const report = (fields: Partial) => reportAttempt(config, runId, { + attempt: implement.attempt, revision: implement.revision, outcome: 'success', driver: owner, ...fields, + }); + + expect(await report({ read: { branch, base: base.commit, commit: base.commit } })) + .toMatchObject({ error: { code: 'invalid_report', message: expect.stringContaining(`was leased branch ${branch}`) } }); + const accepted = await report({ changes: { branch, base: base.commit, from: base.commit, to: commit('c'), commits: [commit('c')] } }); + + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'change.implement' } } }); + expect(accepted).not.toHaveProperty('data.cleanup'); + }); + + it('runs a test in the copy its implement step changed, and records the commit it read', async () => { + const { config, workspace, runId } = await fixture(EXAMPLE); + fs.writeFileSync(path.join(workspace, 'verify.js'), "if (require('fs').readFileSync('tracked.txt', 'utf8') !== 'changed\\n') process.exit(1);\n"); + execFileSync('git', ['add', 'verify.js'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'verify'], { cwd: workspace }); + const base = git(['rev-parse', 'HEAD'], workspace).trim(); + config.workflows = { tools: { default: 'claude' }, max_parallel: 1 }; + const adapter: LocalToolAdapter = { + async runInteractive() { + return { started: false, exitCode: 1, failure: { code: 'tool_failed', message: 'interactive is not used' } }; + }, + async run(_tool, input) { + fs.writeFileSync(path.join(input.cwd, 'tracked.txt'), 'changed\n'); + return { started: true, exitCode: 0 }; + }, + }; + + const done = await runLocalDriver(config, runId, workspace, { adapter }); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const branch = branchName(runId, 'change', 1); + const run = await readRun(config, runId); + const test = typeof run === 'object' ? run.steps.find((step) => step.id === 'change.test') : undefined; + expect(test).toMatchObject({ status: 'succeeded', read: { branch, base, commit: git(['rev-parse', branch], workspace).trim() } }); + expect(test).not.toHaveProperty('changes'); + expect(fs.readFileSync(path.join(workspace, 'tracked.txt'), 'utf8')).toBe('clean\n'); + expect(fs.existsSync(worktreePath(workspace, branch))).toBe(false); + }); + + it.each([ + ['writes a file', `require('fs').writeFileSync('junk.txt','x')`, { code: 'read_only_dirty', message: expect.stringContaining('junk.txt') }], + ['writes a file and exits 1', `require('fs').writeFileSync('junk.txt','x'); process.exit(1)`, undefined], + ['commits a file', `const cp=require('child_process'); require('fs').writeFileSync('junk.txt','x'); cp.execFileSync('git',['add','junk.txt']); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','junk'])`, { code: 'read_only_dirty', message: expect.stringContaining('HEAD') }], + ['commits on a branch it switched to', `const cp=require('child_process'); cp.execFileSync('git',['checkout','-qb','side']); require('fs').writeFileSync('junk.txt','x'); cp.execFileSync('git',['add','junk.txt']); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','junk'])`, { code: 'read_only_dirty', message: expect.stringContaining('HEAD') }], + ])('puts the copy back at the commit a reader read when it %s, so a later step does not commit it', async (_case, script, failure) => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader leaves the copy as it found it. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "${script}" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const test = run.steps.find((step) => step.id === 'change.test'); + const polish = run.steps.find((step) => step.id === 'change.polish'); + expect(test).toMatchObject({ status: 'failed', ...(failure ? { failure } : { message: 'command exited with status 1' }) }); + expect(polish).toMatchObject({ status: 'succeeded' }); + const branch = branchName(runId, 'change', 1); + expect(test && 'read' in test ? test.read?.commit : undefined).toBe(polish && 'changes' in polish ? polish.changes?.from : 'none'); + expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); + expect(() => git(['show', `${branch}:junk.txt`], workspace)).toThrow(); + }); + + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader does not see tool settings committed earlier. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('.tclaude/settings.json','{\\"hooks\\":{}}')" + - id: test + needs: [implement] + command: node -e "if (require('fs').readFileSync('.tclaude/settings.json','utf8')!=='base') process.exit(2)" +`); + fs.mkdirSync(path.join(workspace, '.tclaude')); + fs.writeFileSync(path.join(workspace, '.tclaude', 'settings.json'), 'base'); + execFileSync('git', ['add', '.tclaude'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'tool settings'], { cwd: workspace }); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(git(['show', `${branchName(runId, 'change', 1)}:.tclaude/settings.json`], workspace)).toBe('{"hooks":{}}'); + }); + + it('leases a read-only step outside any copy with no workspace, even when it needs a step that wrote one', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A top-level reader stays in the checkout. +steps: + - id: edit + workspace: new + command: node edit.js + - id: check + needs: [edit] + command: node check.js +`); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const edit = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + const branch = branchName(runId, 'edit', 1); + await reportAttempt(config, runId, { + attempt: edit.attempt, revision: edit.revision, outcome: 'success', driver: owner, + changes: { branch, base: base.commit, from: base.commit, to: commit('c'), commits: [commit('c')] }, + }); + + const check = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + + expect(check.step).toBe('check'); + expect(check).not.toHaveProperty('workspace'); + }); + + it('fails a reader whose needs changed two branches with workspace_base, naming both', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader cannot read two branches at once. +steps: + - id: change + workspace: new + workflow: + steps: + - id: left + workspace: new + command: node left.js + - id: right + needs: [] + workspace: new + command: node right.js + - id: test + needs: [left, right] + command: node verify.js +`); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + for (const [step, to] of [['left', commit('1')], ['right', commit('2')]] as const) { + const leased = leaseOf(await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] })); + expect(leased.step).toBe(`change.${step}`); + await reportAttempt(config, runId, { + attempt: leased.attempt, revision: leased.revision, outcome: 'success', driver: owner, + changes: { branch: branchName(runId, `change.${step}`, 1), base: base.commit, from: base.commit, to, commits: [to] }, + }); + } + + const done = await leaseNext(config, runId, owner, { base, workspaceModes: [...modes] }); + + expect(done).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await readRun(config, runId); + expect(typeof run === 'object' ? run.steps.find((step) => step.id === 'change.test') : run).toMatchObject({ + status: 'failed', + failure: { + code: 'workspace_base', + message: `step "change.test" reads branches ${branchName(runId, 'change.left', 1)}, ${branchName(runId, 'change.right', 1)}; add a step that merges them and need that step`, + }, + }); + }); +}); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index eb922365c..5528f5ac1 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -16,11 +16,14 @@ import { type StepExpression, type TemplateError, } from './template.js'; import { parseStepPath } from './step-path.js'; -import { DriverSchema, answerHitl, cancel, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepState } from './step-state.js'; +import { + DriverSchema, ReadSchema, answerHitl, cancel, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, + type ItemResult, type LeasedStep, type StepRead, type StepState, +} from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { planWorkspace, plannedBranch, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { planWorkspace, plannedBranch, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; import { generateRunId, isRunId, readRun, runLockPath, writeRun } from './store.js'; import type { CachedWorkflow, @@ -61,6 +64,8 @@ export interface ReportInput { failure?: WorkflowFailure; /** The branch this attempt committed, when it ran in a workspace. */ changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; + /** The branch and commit this attempt read, when it ran read-only in a workspace. */ + read?: StepRead; agent?: ReportedAgent; } @@ -996,7 +1001,7 @@ const NEXT_PAYLOAD_FIELDS = new Set(['driver', 'running', 'canTakeLease', 'base' /** The top-level payload fields `parseReport` reads. Keep in sync with the fields it picks. */ const REPORT_FIELDS = new Set([ - 'attempt', 'revision', 'outcome', 'driver', 'outputs', 'exitCode', 'message', 'tool', 'model', 'session', 'tokens', 'failure', 'changes', 'agent', + 'attempt', 'revision', 'outcome', 'driver', 'outputs', 'exitCode', 'message', 'tool', 'model', 'session', 'tokens', 'failure', 'changes', 'read', 'agent', ]); /** @@ -1093,6 +1098,11 @@ export function parseReport(value: unknown): ParsedReport | string { } report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; } + if (record.read !== undefined) { + const read = ReadSchema.safeParse(record.read); + if (!read.success) return 'read must include branch, base, and commit'; + report.read = read.data; + } if (record.agent !== undefined) { const agent = parseReportedAgent(record.agent); if (typeof agent === 'string') return agent; @@ -1159,6 +1169,26 @@ function rememberSettledAttempt(run: RunState, attempt: string): void { if (extra > 0) run.settledAttempts.splice(0, extra); } +/** + * Why a report's `changes` or `read` do not fit the workspace its attempt was leased, or undefined when they fit. + * A step that writes reports `changes`. A read-only step in a copy reports `read`, and a success must carry it. + */ +function workspaceMismatch(report: ReportInput, contract: WorkspaceContract | undefined): string | undefined { + const readOnly = contract?.mode === 'read-only'; + const leased = contract ? `${readOnly ? 'read-only ' : ''}branch ${contract.branch} on base ${contract.base.commit}` : 'no workspace'; + const elsewhere = (branch: string, base: string): boolean => !contract || branch !== contract.branch || base !== contract.base.commit; + if (report.changes && (readOnly || elsewhere(report.changes.branch, report.changes.base))) { + return `changes name branch ${report.changes.branch} on base ${report.changes.base}, but attempt ${report.attempt} was leased ${leased}`; + } + if (report.read && (!readOnly || elsewhere(report.read.branch, report.read.base))) { + return `read names branch ${report.read.branch} on base ${report.read.base}, but attempt ${report.attempt} was leased ${leased}`; + } + if (readOnly && report.outcome === 'success' && !report.read) { + return `attempt ${report.attempt} was leased ${leased}, and its success report has no read`; + } + return undefined; +} + /** A later leaf still runs in `branch`. Workflow and `for_each` steps are not leaves; their children are. */ function branchStillUsed(run: RunState, branch: string, except: string): boolean { for (const item of run.steps) { @@ -1203,18 +1233,11 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: const located = locateStep(run, step.id); const defined = located?.step; - if (report.changes) { - const contract = located && planWorkspace(run, located.scope, step, located.step, { - driver: step.lease.driver, modes: ['read-only', 'new', 'continue'], - }, true).contract; - if (!contract || report.changes.branch !== contract.branch || report.changes.base !== contract.base.commit) { - const leased = contract ? `branch ${contract.branch} on base ${contract.base.commit}` : 'no workspace'; - return { - save: false, - envelope: error('invalid_report', `changes name branch ${report.changes.branch} on base ${report.changes.base}, but attempt ${report.attempt} was leased ${leased}`), - }; - } - } + const contract = located && planWorkspace(run, located.scope, step, located.step, { + driver: step.lease.driver, modes: ['read-only', 'new', 'continue'], + }, true).contract; + const mismatch = workspaceMismatch(report, contract); + if (mismatch) return { save: false, envelope: error('invalid_report', mismatch) }; let outcome = report.outcome; let message = report.message; let failure = report.failure; @@ -1239,6 +1262,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: session: report.session, tokens: report.tokens, ...(report.changes ? { changes: report.changes } : {}), + ...(report.read ? { read: report.read } : {}), agent: settleObservation(report, defined?.agent), }; if (report.changes) { @@ -1249,7 +1273,8 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: const settledStep = outcome === 'released' ? release(step) : outcome === 'success' ? succeed(step, { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }) : fail(step, { ...settlement, failure }); - const cleanup = outcome === 'success' && report.changes !== undefined && !branchStillUsed(run, report.changes.branch, settledStep.id); + // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. + const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch, settledStep.id); return { save: true, envelope: ok({ diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 30af9df75..38009c5c7 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -37,6 +37,7 @@ import { import { driverLockPath, isRunId } from './store.js'; import { stopProcess } from './process-stop.js'; import { chooseStepModel } from './step-model.js'; +import type { StepRead } from './step-state.js'; import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from './types.js'; import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; @@ -320,12 +321,15 @@ function failureReport(failure: WorkflowFailure): { outcome: 'failed'; message: type ReportBody = Parameters[2]; function workspaceOf(value: unknown): WorkspaceContract | undefined { - if (!isRecord(value) || (value.mode !== 'new' && value.mode !== 'continue') || typeof value.branch !== 'string') return undefined; + if (!isRecord(value) || (value.mode !== 'new' && value.mode !== 'continue' && value.mode !== 'read-only') || typeof value.branch !== 'string') return undefined; const base = isRecord(value.base) && typeof value.base.commit === 'string' && (value.base.ref === undefined || typeof value.base.ref === 'string') ? { ...(typeof value.base.ref === 'string' ? { ref: value.base.ref } : {}), commit: value.base.commit } : undefined; const start = isRecord(value.start) && typeof value.start.commit === 'string' ? { commit: value.start.commit, ...(typeof value.start.branch === 'string' ? { branch: value.start.branch } : {}) } : undefined; if (!base || !start || typeof value.configCommit !== 'string' || !Array.isArray(value.merge)) return undefined; + if (value.mode === 'read-only') { + return value.merge.length === 0 ? { mode: value.mode, base, configCommit: value.configCommit, branch: value.branch, start, merge: [] } : undefined; + } const merge = value.merge.flatMap((item) => ( isRecord(item) && typeof item.branch === 'string' && typeof item.commit === 'string' ? [{ branch: item.branch, commit: item.commit }] : [] )); @@ -357,7 +361,7 @@ function parseLease(raw: unknown): Lease | undefined { return (lease.command === undefined) === (lease.prompt === undefined) ? undefined : lease; } -/** The checkout this lease uses. Read-only steps share the active checkout; a workspace has its own worktree. */ +/** The checkout this lease uses. A step without a workspace shares the active checkout; a workspace has its own worktree. */ function placeOf(lease: Lease, active: string): string { return lease.cwd ?? (lease.workspace ? worktreePath(active, lease.workspace.branch) : active); } @@ -399,6 +403,13 @@ async function stageLease(active: string, lease: Lease): Promise { lease.overlay = undefined; } +/** + * Leaves a read-only copy as the step found it: the base agent files go back, and a copy the step changed is reset to + * the commit it read, so a later step there does not commit the change. Undefined for any other lease. + */ +async function leaveReadCopy(lease: Lease): Promise<{ read: StepRead; changed: string[] } | undefined> { + const { cwd, before, workspace } = lease; + if (!cwd || !before?.head || workspace?.mode !== 'read-only') return undefined; + const changed = changedPaths(before, await snapshotWorktree(cwd)); + await restoreLeaseOverlay(lease); + if (changed.length > 0) { + // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. + await git(cwd, ['checkout', '-q', '-f', workspace.branch]); + await git(cwd, ['reset', '-q', '--hard', before.head]); + await git(cwd, ['clean', '-q', '-fd']); + } + return { read: { branch: workspace.branch, base: workspace.base.commit, commit: before.head }, changed }; +} + async function finishWorkspace(lease: Lease, push: boolean): Promise<{ changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; failure?: WorkflowFailure }> { - if (!lease.workspace || !lease.cwd) return {}; + if (!lease.workspace || lease.workspace.mode === 'read-only' || !lease.cwd) return {}; if (lease.overlay) await restoreUneditedConfig(lease.cwd, lease.workspace.configCommit, lease.overlay); lease.overlay = undefined; const open = await unresolvedConflicts(lease.cwd, lease.conflicts); @@ -543,9 +572,14 @@ async function commandReport(attemptResultPath: string, driver: DriverIdentity, return { attempt: lease.attempt, revision: lease.revision, driver, exitCode: 0, ...failureReport(failure) }; } } - const finished = lease.workspace ? await finishWorkspace(lease, result.exitCode === 0) : {}; - const failed = result.exitCode !== 0 || finished.failure; - const message = finished.failure?.message + const left = await leaveReadCopy(lease); + const dirty = left && result.exitCode === 0 && left.changed.length > 0 + ? { code: 'read_only_dirty', message: `Read-only command step changed repository paths: ${left.changed.join(', ')}. Workspace ${left.read.branch} was put back at ${left.read.commit}.` } + : undefined; + const finished = lease.workspace && !left ? await finishWorkspace(lease, result.exitCode === 0) : {}; + const failure = dirty ?? finished.failure; + const failed = result.exitCode !== 0 || failure; + const message = failure?.message ?? (result.exitCode !== 0 ? (snippet === '' ? `command exited with status ${result.exitCode}` : `command exited with status ${result.exitCode}: ${snippet}`) : undefined); return { attempt: lease.attempt, @@ -555,8 +589,9 @@ async function commandReport(attemptResultPath: string, driver: DriverIdentity, ...(output !== undefined ? { outputs: output } : {}), exitCode: result.exitCode, ...(message ? { message } : {}), - ...(finished.failure ? { failure: finished.failure } : {}), + ...(failure ? { failure } : {}), ...(finished.changes ? { changes: finished.changes } : {}), + ...(left ? { read: left.read } : {}), }; } @@ -640,11 +675,12 @@ async function promptReport( failure: { code: 'tool_failed', message: `${selected} could not complete the workflow step` }, }; } - const paths = before && !lease.workspace ? changedPaths(before, await snapshotWorktree(cwd)) : []; - const dirtyFailure = paths.length > 0 - ? { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}` } - : undefined; - const finished = lease.workspace && !dirtyFailure ? await finishWorkspace(lease, result.exitCode === 0 && !result.failure) : {}; + const left = await leaveReadCopy(lease); + const paths = left ? left.changed : before && !lease.workspace ? changedPaths(before, await snapshotWorktree(cwd)) : []; + const dirtyFailure = paths.length === 0 ? undefined : left + ? { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}. Workspace ${left.read.branch} was put back at ${left.read.commit}.` } + : { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}` }; + const finished = lease.workspace && !left && !dirtyFailure ? await finishWorkspace(lease, result.exitCode === 0 && !result.failure) : {}; const failure = dirtyFailure ?? finished.failure ?? result.failure; const ran = result.started ? result : undefined; return { @@ -660,6 +696,7 @@ async function promptReport( ...(ran ? { agent: ran.applied ? { ...observed, definition: { applied: ran.applied } } : observed } : {}), exitCode: result.exitCode, ...(finished.changes ? { changes: finished.changes } : {}), + ...(left ? { read: left.read } : {}), ...(failure ? failureReport(failure) : { outcome: result.exitCode === 0 ? 'success' : 'failed' }), }; } @@ -894,7 +931,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options if (exited.kind === 'stopped') { const stopping = dying.get(exited.attempt); dying.delete(exited.attempt); - if (stopping) await restoreLeaseOverlay(stopping.lease); + if (stopping) await (stopping.lease.workspace?.mode === 'read-only' ? leaveReadCopy(stopping.lease) : restoreLeaseOverlay(stopping.lease)); continue; } inFlight.delete(exited.attempt); @@ -908,7 +945,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options for (const attempt of inFlight.keys()) cancel(attempt); await Promise.all([...dying.values()].map(async ({ lease, command }) => { await command.stop(); - await restoreLeaseOverlay(lease); + await (lease.workspace?.mode === 'read-only' ? leaveReadCopy(lease) : restoreLeaseOverlay(lease)); })); if (signalled) exitOn(signalled); for (const name of STOP_SIGNALS) process.off(name, onSignal); diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 89dfa010b..f249d9864 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -23,6 +23,14 @@ const ChangesSchema = z.object({ commits: z.array(z.string()), }); +/** What a read-only step inside a copy reports: the branch it ran on and the commit it read. It wrote nothing. */ +export const ReadSchema = z.object({ + branch: z.string().min(1), + base: z.string().min(1), + commit: z.string().min(1), +}); +export type StepRead = z.infer; + /** What a driver reports about the attempt that settled a step. */ const report = { exitCode: z.number().optional(), @@ -32,6 +40,7 @@ const report = { session: z.string().optional(), tokens: TokenUsageSchema.optional(), changes: ChangesSchema.optional(), + read: ReadSchema.optional(), /** * What the pass ran as (#1176), settled by the engine before it is saved. One this CLI cannot read, such as a newer * version's, reads as absent rather than making the run unreadable. diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index a88156f28..15c854ba2 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -19,16 +19,20 @@ export interface StepChanges { commits: string[]; } -/** The git contract on a `new` or `continue` lease. Paths stay off it. */ -export interface WorkspaceContract { - mode: 'new' | 'continue'; +/** + * The git contract on a lease that runs in a copy. Paths stay off it. A `new` or `continue` step commits to `branch`. + * A `read-only` step inside a copy runs in `branch`'s worktree, merges and commits nothing, and leaves it as it was. + */ +export type WorkspaceContract = { base: { ref?: string; commit: string }; /** The commit agent configuration is restored from before the step runs. */ configCommit: string; branch: string; start: { commit: string; branch?: string }; - merge: Array<{ branch: string; commit: string }>; -} +} & ( + | { mode: 'new' | 'continue'; merge: Array<{ branch: string; commit: string }> } + | { mode: 'read-only'; merge: [] } +); export interface SessionRef { tool: string; @@ -56,7 +60,7 @@ export function branchName(runId: string, stepPath: string, pass: number): strin } /** - * How a step runs. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy. + * How a step runs. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy, which it reads. * A prompt that omits it continues the nearest enclosing copy, or stays read-only. A `for_each` copy is `new` * per item, so siblings do not share a worktree. */ @@ -138,6 +142,32 @@ function mergeChanges(run: RunState, scope: Scope, state: StepState, definition: return ordered.map((item) => item.changes); } +/** + * The branches a read-only step's own `needs` wrote or read, each with the newest commit a need left or read there. + * Inherited needs and `after` items are where a copy starts, not what a reader reads, so they are left out. + */ +function readHeads(run: RunState, scope: Scope, definition: WorkflowStep): Array<{ branch: string; commit: string; settledAt: number }> { + const heads: Array<{ branch: string; commit: string; settledAt: number }> = []; + const visit = (step: StepState | undefined): void => { + if (!step) return; + const changes = changesOf(step); + const read = 'read' in step ? step.read : undefined; + const head = changes ? { branch: changes.branch, commit: changes.to } : read; + if (!head) { + for (const item of itemsOf(run, step)) visit(item); + for (const child of workflowChildren(run, step)) visit(child); + return; + } + const settledAt = 'settledAt' in step ? step.settledAt : 0; + const earlier = heads.find((known) => known.branch === head.branch); + if (!earlier) heads.push({ branch: head.branch, commit: head.commit, settledAt }); + else if (settledAt > earlier.settledAt) Object.assign(earlier, { commit: head.commit, settledAt }); + }; + const states = statesIn(run, scope); + for (const id of definition.needs ?? []) visit(states.find((item) => localId(scope, item) === id)); + return heads; +} + /** Item ids on a step path, such as `fan[1]` inside `fan[1].check`. */ function itemIdsIn(id: string): string[] { const ids: string[] = []; @@ -239,9 +269,34 @@ export function planWorkspace( const mode = resolveMode(definition, ancestors.map((ancestor) => ancestor.step)); const sessionResult = sessionOf(run, scope, state, definition); if (sessionResult && 'code' in sessionResult) return { mode, fail: sessionResult }; - if (mode === 'read-only') return { mode, ...(sessionResult ? { session: sessionResult } : {}) }; - + const session = sessionResult ? { session: sessionResult } : {}; + const owner = [...ancestors].reverse().find((ancestor) => ancestor.step.workspace === 'new' + || (ancestor.step.forEach !== undefined && ancestor.step.workspace === 'continue')); const modes = request.modes ?? ['read-only']; + if (mode === 'read-only') { + if (!owner) return { mode, ...session }; + // A reader runs where its need left the change, or in the copy a `continue` step there would use. + const heads = readHeads(run, scope, definition); + if (heads.length > 1) { + const branches = heads.map((head) => head.branch).join(', '); + return { mode, fail: { code: 'workspace_base', message: `step "${state.id}" reads branches ${branches}; add a step that merges them and need that step` } }; + } + const needed = heads.length === 1 ? heads[0] : undefined; + const branch = needed?.branch ?? branchName(run.id, owner.instance, owner.pass); + if (!handback && !modes.includes('continue')) { + return { mode, defer: `step "${state.id}" reads workspace ${branch}, which needs continue` }; + } + const base = run.base ?? request.base; + if (!base) { + return { mode, fail: { code: 'workspace_base', message: `step "${state.id}" needs the run's base commit` } }; + } + const copy = run.copies?.[branch]; + if (!handback && copy && !sameDriver(copy.owner, request.driver)) { + return { mode, defer: `step "${state.id}" reads a workspace owned by another driver` }; + } + const start = { commit: copy?.commit ?? needed?.commit ?? base.commit }; + return { mode, ...session, contract: { mode, base, configCommit: base.commit, branch, start, merge: [] } }; + } if (!handback && !modes.includes(mode)) { return { mode, defer: `step "${state.id}" needs workspace ${mode}, which this driver does not provide` }; } @@ -249,8 +304,6 @@ export function planWorkspace( if (!base) { return { mode, fail: { code: 'workspace_base', message: `step "${state.id}" needs the run's base commit` } }; } - const owner = [...ancestors].reverse().find((ancestor) => ancestor.step.workspace === 'new' - || (ancestor.step.forEach !== undefined && ancestor.step.workspace === 'continue')); const branch = mode === 'new' || !owner ? branchName(run.id, state.id, state.pass ?? 1) : branchName(run.id, owner.instance, owner.pass); From ba955d66a207d39cbb6ddae6a7d5cdae04002fb2 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 02:39:51 +0200 Subject: [PATCH 09/41] fix(workflow): refuse agent paths under a symlink and keep .github workflows A branch can commit .github as a symlink. The nested Copilot paths were then copied and removed through it, deleting files outside the worktree. A root-level hooks file no longer yields '.' as a path to reset. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-workspace.test.ts | 76 ++++++++++++++++++++++++ src/workflow/worktree.ts | 21 ++++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index c29c5dbea..22436a4b6 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy, it runs `git reset --hard` to the commit it read and `git clean -fd`, so a later writer never commits a reader's edits. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy, it runs `git reset --hard` to the commit it read and `git clean -fd`, so a later writer never commits a reader's edits. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 386979f2c..7584e9518 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -690,6 +690,82 @@ steps: expect(git(['show', `${branch}:.tclaude/settings.json`], workspace)).toBe('earlier'); }); + it('fails a later step with base_config when an earlier step committed .github as a symlink, and leaves its target alone', async () => { + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-outside-')); + dirs.push(outside); + fs.mkdirSync(path.join(outside, 'skills')); + fs.writeFileSync(path.join(outside, 'skills', 'keep.txt'), 'keep'); + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: An agent path under a symlinked parent is not touched. +steps: + - id: work + workspace: new + workflow: + steps: + - id: plant + workspace: continue + command: node -e "require('fs').symlinkSync(${JSON.stringify(outside).replace(/"/g, "'")},'.github')" + - id: check + needs: [plant] + workspace: continue + command: node -e "require('fs').writeFileSync('ok.txt','yes')" +`); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await readRun(config, runId); + const check = typeof run === 'object' ? run.steps.find((step) => step.id === 'work.check') : undefined; + expect(check).toMatchObject({ status: 'failed', failure: { code: 'base_config', message: expect.stringContaining('.github') } }); + expect(fs.readFileSync(path.join(outside, 'skills', 'keep.txt'), 'utf8')).toBe('keep'); + }); + + it('keeps .github workflows from the branch and resets only its agent paths', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A later step sees the workflows an earlier step committed, not its hooks. +steps: + - id: work + workspace: new + workflow: + steps: + - id: plant + workspace: continue + command: node -e "const fs=require('fs'); fs.mkdirSync('.github/workflows',{recursive:true}); fs.mkdirSync('.github/hooks',{recursive:true}); fs.writeFileSync('.github/workflows/ci.yml','ci'); fs.writeFileSync('.github/hooks/x.json','{}')" + - id: check + needs: [plant] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('.github/workflows/ci.yml') || fs.existsSync('.github/hooks/x.json')) process.exit(2); fs.writeFileSync('ok.txt','yes')" +`); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await readRun(config, runId); + const check = typeof run === 'object' ? run.steps.find((step) => step.id === 'work.check') : undefined; + const branch = check && 'changes' in check ? check.changes?.branch : 'none'; + expect(git(['show', `${branch}:ok.txt`], workspace)).toBe('yes'); + expect(git(['show', `${branch}:.github/workflows/ci.yml`], workspace)).toBe('ci'); + expect(git(['show', `${branch}:.github/hooks/x.json`], workspace)).toBe('{}'); + }); + + it('never resets the whole worktree for a hooks file at the repository root', async () => { + vi.resetModules(); + vi.doMock('../types.js', () => ({ + TeamaiConfigBaseSchema: { shape: { toolPaths: { parse: () => ({ tool: { hooks: 'hooks.json', skills: '.tool/skills' } }) } } }, + })); + try { + const { AGENT_PATHS } = await import('../workflow/worktree.js'); + expect(AGENT_PATHS).toContain('hooks.json'); + expect(AGENT_PATHS).not.toContain('.'); + expect(AGENT_PATHS).not.toContain(''); + } finally { + vi.doUnmock('../types.js'); + vi.resetModules(); + } + }); + it('delivers a team skill into a new workspace and leaves it out of the branch', async () => { const { config, workspace, runId } = await teamWorkspace(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 68f111a09..aaa9efdc4 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -18,7 +18,10 @@ function agentConfigPaths(): string[] { for (const tool of Object.values(TeamaiConfigBaseSchema.shape.toolPaths.parse(undefined))) { for (const field of ['skills', 'rules', 'settings', 'hooks', 'claudemd', 'agents', 'mcpProject'] as const) { const relative = tool[field]; - if (relative) paths.add(configRoot(field === 'hooks' ? path.posix.dirname(relative) : relative)); + if (!relative) continue; + const root = field === 'hooks' ? path.posix.dirname(relative) : relative; + // A hooks file at the repository root has dirname `.`, which would reset the whole worktree. + paths.add(configRoot(root === '.' ? relative : root)); } } return [...paths]; @@ -155,6 +158,10 @@ export async function applyBaseConfig( dir: string, commit: string, hooks: string, ): Promise<{ ok: true; saved: string } | { ok: false; message: string }> { await git(dir, ['config', '--worktree', 'core.hooksPath', hooks]); + const linked = await symlinkedParent(dir); + if (linked) { + return { ok: false, message: `Agent configuration ${linked.relative} is under the symlink ${linked.parent} on this branch, so it was not reset. Commit ${linked.parent} as a directory on the branch and retry the step.` }; + } const saved = await fs.mkdtemp(path.join(os.tmpdir(), 'teamai-agent-')); const conflicts = (await git(dir, ['ls-files', '-u', '-z', '--', ...AGENT_PATHS])).stdout; await fs.writeFile(path.join(saved, '_conflicts'), conflicts); @@ -186,6 +193,18 @@ export async function applyBaseConfig( return { ok: true, saved }; } +/** An agent path whose parent is a symlink. Copying or removing it would reach outside the worktree. */ +async function symlinkedParent(dir: string): Promise<{ relative: string; parent: string } | undefined> { + for (const relative of AGENT_PATHS) { + const parts = relative.split('/'); + for (let end = 1; end < parts.length; end += 1) { + const parent = parts.slice(0, end).join('/'); + if (await fs.lstat(path.join(dir, parent)).then((stats) => stats.isSymbolicLink(), () => false)) return { relative, parent }; + } + } + return undefined; +} + /** Puts saved conflict stages back after the base checkout cleared them, before the step runs. */ export async function reopenConfigConflicts(dir: string, saved: string): Promise { const conflicts = await fs.readFile(path.join(saved, '_conflicts'), 'utf8').catch(() => ''); From fbc539f19d2fa45bd8f3563e1fec518e2f51cd6f Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 02:44:15 +0200 Subject: [PATCH 10/41] fix(workflow): put a read-only copy back on its branch and keep what the reader changed A reader that detached HEAD changed nothing the snapshot measured, so a later continue step committed off the branch. A reset also dropped a person's uncommitted edits, ignored git failures while reporting the copy as put back, and left initialized submodules where the reader moved them. The state is now kept on refs/teamai/kept/ (not the stash list, which every worktree shares) before the reset. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow-advanced.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow-advanced.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 94 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 46 +++++++++--- 7 files changed, 134 insertions(+), 16 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 22436a4b6..51b42bc90 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy, it runs `git reset --hard` to the commit it read and `git clean -fd`, so a later writer never commits a reader's edits. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, `git stash create`; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and the same for initialized submodules, so a later writer never commits a reader's edits. A git failure there fails `workspace_base` and says the copy was not put back. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow-advanced.md b/docs/guide/workflow-advanced.md index bc5d321c4..cfb100501 100644 --- a/docs/guide/workflow-advanced.md +++ b/docs/guide/workflow-advanced.md @@ -632,7 +632,7 @@ Without that capability, a `next` request that can take a lease applies this tab Every decision without HITL and `hitl_required` failure logs a warning to stderr. Set `workflows.auto_approve` beside `workflows.max_parallel` in local config, or pass `--auto-approve` to `start --driver local` or `resume` for that process only. JSON `next` cannot set this override. The local driver declares HITL only when stdin and stderr are TTYs. It asks on stderr, accepts a choice followed by an optional note, and keeps polling while the question is open. An answer from elsewhere or a cancellation closes that question. Questions wait while an interactive tool owns the terminal. -`interactive: true` needs `prompt` or `prompt_file`. A driver without HITL fails it with `hitl_required`. The local driver opens the tool here in the workspace with inherited stdio and the prompt as its first message. Repository edits by the person are kept, and the step succeeds. An interactive step that sets `workspace: read-only` is checked like a command: a checkout change fails `read_only_dirty` and the edits stay. An interactive step in its own workspace can still change that workspace. Headless prompts remain read-only. Terminal Ctrl-C and Ctrl-\ go to the interactive tool; SIGTERM and SIGHUP stop it and the driver. A result file is optional when the interactive tool exits successfully without outputs. The driver polls every two seconds during prompts and stops a cancelled tool. +`interactive: true` needs `prompt` or `prompt_file`. A driver without HITL fails it with `hitl_required`. The local driver opens the tool here in the workspace with inherited stdio and the prompt as its first message. Repository edits by the person are kept, and the step succeeds. An interactive step that sets `workspace: read-only` is checked like a command: a checkout change fails `read_only_dirty` and the edits stay. Inside a workspace an enclosing step opened, they are kept on `refs/teamai/kept/` and removed from the workspace. An interactive step in its own workspace can still change that workspace. Headless prompts remain read-only. Terminal Ctrl-C and Ctrl-\ go to the interactive tool; SIGTERM and SIGHUP stop it and the driver. A result file is optional when the interactive tool exits successfully without outputs. The driver polls every two seconds during prompts and stops a cancelled tool. ## Retry, cancel, and resume diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index f227ee11d..8f6f25031 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -139,7 +139,7 @@ Use `workflow respond ` for a waiting HITL request, `workfl A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. -Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it, the worktree is reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. +Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, the worktree and its submodules are reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. If git cannot put it back, the step fails with `workspace_base`. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. `workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. diff --git a/docs/guide/zh-CN/workflow-advanced.md b/docs/guide/zh-CN/workflow-advanced.md index 959bbcb9a..c47e58172 100644 --- a/docs/guide/zh-CN/workflow-advanced.md +++ b/docs/guide/zh-CN/workflow-advanced.md @@ -632,7 +632,7 @@ teamai workflow resume 每次无 HITL 决策和 hitl_required 失败都会向 stderr 写警告。在本地配置中与 `workflows.max_parallel` 一起设置 `workflows.auto_approve`，或给 `start --driver local`、`resume` 传 `--auto-approve`，仅对当前进程生效。JSON next 不能设置此覆盖。本地驱动仅在 stdin 和 stderr 都是 TTY 时声明支持 HITL，通过 stderr 提问，接受选择及其后的可选备注，并在提问期间继续轮询。其他进程回答或取消 HITL 请求会关闭问题。交互工具占用终端时，HITL 请求提问会等待。 -`interactive: true` 要求 `prompt` 或 `prompt_file`。没有 HITL 的驱动以 `hitl_required` 失败。本地驱动在工作目录以前台方式打开工具，继承 stdio，并把 prompt 作为第一条消息。人对仓库的修改会保留，步骤成功。设置了 `workspace: read-only` 的交互步骤和命令一样检查：改了检出目录就以 `read_only_dirty` 失败，改动留在原地。在自己工作区里的交互步骤仍可以改那个工作区。无头 prompt 仍然只读。终端的 Ctrl-C 和 Ctrl-\ 交给交互工具处理；SIGTERM 和 SIGHUP 会停止工具和驱动。交互工具以 0 退出且没有输出时可以不写结果文件。驱动在 prompt 运行时每两秒轮询，发现取消后停止工具。 +`interactive: true` 要求 `prompt` 或 `prompt_file`。没有 HITL 的驱动以 `hitl_required` 失败。本地驱动在工作目录以前台方式打开工具，继承 stdio，并把 prompt 作为第一条消息。人对仓库的修改会保留，步骤成功。设置了 `workspace: read-only` 的交互步骤和命令一样检查：改了检出目录就以 `read_only_dirty` 失败，改动留在原地。在外层步骤打开的工作区里，改动保存在 `refs/teamai/kept/`，并从工作区移除。在自己工作区里的交互步骤仍可以改那个工作区。无头 prompt 仍然只读。终端的 Ctrl-C 和 Ctrl-\ 交给交互工具处理；SIGTERM 和 SIGHUP 会停止工具和驱动。交互工具以 0 退出且没有输出时可以不写结果文件。驱动在 prompt 运行时每两秒轮询，发现取消后停止工具。 ## 重试、取消和继续运行 diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 8749e0604..563f07b06 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -139,7 +139,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom 没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 -在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区时，worktree 重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 +在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。git 无法恢复工作区时，步骤以 `workspace_base` 失败。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 `workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 7584e9518..1cf56e8a1 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2375,6 +2375,7 @@ steps: ['writes a file and exits 1', `require('fs').writeFileSync('junk.txt','x'); process.exit(1)`, undefined], ['commits a file', `const cp=require('child_process'); require('fs').writeFileSync('junk.txt','x'); cp.execFileSync('git',['add','junk.txt']); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','junk'])`, { code: 'read_only_dirty', message: expect.stringContaining('HEAD') }], ['commits on a branch it switched to', `const cp=require('child_process'); cp.execFileSync('git',['checkout','-qb','side']); require('fs').writeFileSync('junk.txt','x'); cp.execFileSync('git',['add','junk.txt']); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','junk'])`, { code: 'read_only_dirty', message: expect.stringContaining('HEAD') }], + ['detaches HEAD', `require('child_process').execFileSync('git',['checkout','-q','--detach','HEAD'])`, { code: 'read_only_dirty', message: expect.stringContaining('HEAD') }], ])('puts the copy back at the commit a reader read when it %s, so a later step does not commit it', async (_case, script, failure) => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build @@ -2411,6 +2412,99 @@ steps: expect(() => git(['show', `${branch}:junk.txt`], workspace)).toThrow(); }); + it('keeps what a reader changed on a ref named in its failure before putting the copy back', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader's edits are recoverable after the copy is put back. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').writeFileSync('junk.txt','reader')" +`); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + const test = typeof run === 'object' ? run.steps.find((step) => step.id === 'change.test') : undefined; + const kept = `refs/teamai/kept/${test && 'attempt' in test ? test.attempt : 'none'}`; + expect(test).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining(kept) } }); + expect(git(['show', `${kept}:junk.txt`], workspace)).toBe('reader'); + expect(() => git(['show', `${branchName(runId, 'change', 1)}:junk.txt`], workspace)).toThrow(); + }); + + it('fails a reader with workspace_base, and does not say the copy was put back, when git cannot put it back', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader that deletes its copy's branch leaves a copy that cannot be put back. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'); const b=cp.execFileSync('git',['symbolic-ref','--short','HEAD']).toString().trim(); cp.execFileSync('git',['checkout','-q','--detach']); cp.execFileSync('git',['branch','-qD',b])" +`); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + const test = typeof run === 'object' ? run.steps.find((step) => step.id === 'change.test') : undefined; + expect(test).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('could NOT be put back') } }); + expect(test).not.toMatchObject({ failure: { message: expect.stringContaining('was put back') } }); + }); + + it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader's commit and untracked file inside a submodule do not reach the branch. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'),fs=require('fs'); fs.writeFileSync('vendor/inside.txt','reader'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd:'vendor'}); fs.writeFileSync('vendor/untracked.txt','x')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inside.txt','utf8')!=='one\\\\n' || fs.existsSync('vendor/untracked.txt')) process.exit(2); fs.writeFileSync('polish.txt','yes')" +`); + const source = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); + dirs.push(source); + execFileSync('git', ['init', '-q'], { cwd: source }); + fs.writeFileSync(path.join(source, 'inside.txt'), 'one\n'); + execFileSync('git', ['add', 'inside.txt'], { cwd: source }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: source }); + execFileSync('git', ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', source, 'vendor'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'vendor'], { cwd: workspace }); + const recorded = git(['rev-parse', 'HEAD:vendor'], workspace).trim(); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('vendor') } }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); + const branch = branchName(runId, 'change', 1); + expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); + expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); + }); + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 38009c5c7..f84b5fdf8 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -522,22 +522,46 @@ async function restoreLeaseOverlay(lease: Lease): Promise { lease.overlay = undefined; } +type ReadCopy = { read: StepRead; changed: string[]; kept?: string; unrestored?: { error: string; dir: string } }; + /** * Leaves a read-only copy as the step found it: the base agent files go back, and a copy the step changed is reset to - * the commit it read, so a later step there does not commit the change. Undefined for any other lease. + * the commit it read, so a later step there does not commit the change. What the step changed is kept on a ref first. + * Undefined for any other lease. */ -async function leaveReadCopy(lease: Lease): Promise<{ read: StepRead; changed: string[] } | undefined> { +async function leaveReadCopy(lease: Lease): Promise { const { cwd, before, workspace } = lease; if (!cwd || !before?.head || workspace?.mode !== 'read-only') return undefined; const changed = changedPaths(before, await snapshotWorktree(cwd)); + // A detach or a switch to another branch at the same commit changes no file, but a later step would commit off the branch. + const onBranch = (await git(cwd, ['symbolic-ref', '--quiet', '--short', 'HEAD'])).stdout.trim() === workspace.branch; + if (!onBranch && !changed.includes('HEAD')) changed.push('HEAD'); await restoreLeaseOverlay(lease); - if (changed.length > 0) { - // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. - await git(cwd, ['checkout', '-q', '-f', workspace.branch]); - await git(cwd, ['reset', '-q', '--hard', before.head]); - await git(cwd, ['clean', '-q', '-fd']); + const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; + if (changed.length === 0) return { read, changed }; + // Not `git stash push`: the stash list is shared by every worktree of the checkout. + const kept = `refs/teamai/kept/${lease.attempt}`; + const added = await git(cwd, ['add', '-A']); + const stashed = added.code === 0 ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'stash', 'create']) : added; + const recorded = stashed.code === 0 ? await git(cwd, ['update-ref', kept, stashed.stdout.trim() || 'HEAD']) : stashed; + // Without the ref, a reset would lose the changes. + if (recorded.code !== 0) return { read, changed, unrestored: { error: recorded.stderr.trim() || 'its changes could not be kept', dir: cwd } }; + // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. + // A superproject reset leaves initialized submodules where the step moved them. + const submodules = [['submodule', '--quiet', 'update', '--recursive', '--force'], ['submodule', '--quiet', 'foreach', '--recursive', 'git clean -q -fd']]; + for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ...submodules]) { + const restored = await git(cwd, args); + if (restored.code !== 0) return { read, changed, kept, unrestored: { error: restored.stderr.trim() || `git ${args.join(' ')} failed`, dir: cwd } }; } - return { read: { branch: workspace.branch, base: workspace.base.commit, commit: before.head }, changed }; + return { read, changed, kept }; +} + +/** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ +function readCopyFailure(left: ReadCopy, changed: string): WorkflowFailure { + const kept = left.kept ? ` Its changes are kept on ${left.kept}.` : ''; + return left.unrestored + ? { code: 'workspace_base', message: `${changed} Workspace ${left.read.branch} could NOT be put back at ${left.read.commit}: ${left.unrestored.error}.${kept} Put ${left.unrestored.dir} back on ${left.read.branch} at ${left.read.commit} before a later step runs there.` } + : { code: 'read_only_dirty', message: `${changed} Workspace ${left.read.branch} was put back at ${left.read.commit}.${kept}` }; } async function finishWorkspace(lease: Lease, push: boolean): Promise<{ changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; failure?: WorkflowFailure }> { @@ -573,8 +597,8 @@ async function commandReport(attemptResultPath: string, driver: DriverIdentity, } } const left = await leaveReadCopy(lease); - const dirty = left && result.exitCode === 0 && left.changed.length > 0 - ? { code: 'read_only_dirty', message: `Read-only command step changed repository paths: ${left.changed.join(', ')}. Workspace ${left.read.branch} was put back at ${left.read.commit}.` } + const dirty = left && (result.exitCode === 0 || left.unrestored) && left.changed.length > 0 + ? readCopyFailure(left, `Read-only command step changed repository paths: ${left.changed.join(', ')}.`) : undefined; const finished = lease.workspace && !left ? await finishWorkspace(lease, result.exitCode === 0) : {}; const failure = dirty ?? finished.failure; @@ -678,7 +702,7 @@ async function promptReport( const left = await leaveReadCopy(lease); const paths = left ? left.changed : before && !lease.workspace ? changedPaths(before, await snapshotWorktree(cwd)) : []; const dirtyFailure = paths.length === 0 ? undefined : left - ? { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}. Workspace ${left.read.branch} was put back at ${left.read.commit}.` } + ? readCopyFailure(left, `Read-only prompt step changed repository paths: ${paths.join(', ')}.`) : { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}` }; const finished = lease.workspace && !left && !dirtyFailure ? await finishWorkspace(lease, result.exitCode === 0 && !result.failure) : {}; const failure = dirtyFailure ?? finished.failure ?? result.failure; From 662c5fce02285ab763cde7f8cf8e9e1da012fcfb Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:04:58 +0200 Subject: [PATCH 11/41] fix(workflow): say a reader's changes were NOT saved when keeping them fails When update-ref or stash create failed, the failure said the copy could NOT be put back and told the user to put it back, but not that the edits exist only in the copy. Read literally, that is a reset that destroys them. The result of leaving a read copy is now a tagged union (untouched, restored, unrestored), so each message follows from one state. --- src/__tests__/workflow-workspace.test.ts | 50 ++++++++++++++++++----- src/workflow/local-driver.ts | 51 ++++++++++++++++-------- 2 files changed, 76 insertions(+), 25 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 1cf56e8a1..b562d8678 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -111,6 +111,19 @@ async function teamWorkspace(yaml: string): Promise<{ config: LocalConfig; works return { config, workspace, runId: String(started.data.run) }; } +/** Commits a submodule `vendor` holding `inside.txt` (`one`) to the workspace, and returns the commit it records. */ +function addSubmodule(workspace: string): string { + const source = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); + dirs.push(source); + execFileSync('git', ['init', '-q'], { cwd: source }); + fs.writeFileSync(path.join(source, 'inside.txt'), 'one\n'); + execFileSync('git', ['add', 'inside.txt'], { cwd: source }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: source }); + execFileSync('git', ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', source, 'vendor'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'vendor'], { cwd: workspace }); + return git(['rev-parse', 'HEAD:vendor'], workspace).trim(); +} + const driver = (workspace: string, machine = 'host'): DriverIdentity => ({ kind: 'local', machine, workspace }); afterEach(() => { @@ -2439,6 +2452,33 @@ steps: expect(() => git(['show', `${branchName(runId, 'change', 1)}:junk.txt`], workspace)).toThrow(); }); + it('says the changes were NOT saved, and leaves them in the copy, when they cannot be kept on a ref', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A ref in the way of refs/teamai/kept/ stops the keep. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').writeFileSync('junk.txt','reader')" +`); + git(['update-ref', 'refs/teamai/kept', 'HEAD'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + const test = typeof run === 'object' ? run.steps.find((step) => step.id === 'change.test') : undefined; + const copy = worktreePath(workspace, branchName(runId, 'change', 1)); + expect(test).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`Its changes were NOT saved and are still in ${copy}`) } }); + expect(fs.readFileSync(path.join(copy, 'junk.txt'), 'utf8')).toBe('reader'); + }); + it('fails a reader with workspace_base, and does not say the copy was put back, when git cannot put it back', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build @@ -2484,15 +2524,7 @@ steps: workspace: continue command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inside.txt','utf8')!=='one\\\\n' || fs.existsSync('vendor/untracked.txt')) process.exit(2); fs.writeFileSync('polish.txt','yes')" `); - const source = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); - dirs.push(source); - execFileSync('git', ['init', '-q'], { cwd: source }); - fs.writeFileSync(path.join(source, 'inside.txt'), 'one\n'); - execFileSync('git', ['add', 'inside.txt'], { cwd: source }); - execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: source }); - execFileSync('git', ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', source, 'vendor'], { cwd: workspace }); - execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'vendor'], { cwd: workspace }); - const recorded = git(['rev-parse', 'HEAD:vendor'], workspace).trim(); + const recorded = addSubmodule(workspace); await runLocalDriver(config, runId, workspace); diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index f84b5fdf8..fbd537028 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -522,7 +522,12 @@ async function restoreLeaseOverlay(lease: Lease): Promise { lease.overlay = undefined; } -type ReadCopy = { read: StepRead; changed: string[]; kept?: string; unrestored?: { error: string; dir: string } }; +/** How a reader left its copy. `kept` is the ref holding what the step changed; an unrestored copy lacks it when keeping failed. */ +type ReadCopy = { read: StepRead } & ( + | { state: 'untouched' } + | { state: 'restored'; changed: string[]; kept: string } + | { state: 'unrestored'; changed: string[]; reason: string; dir: string; kept?: string } +); /** * Leaves a read-only copy as the step found it: the base agent files go back, and a copy the step changed is reset to @@ -538,30 +543,42 @@ async function leaveReadCopy(lease: Lease): Promise { if (!onBranch && !changed.includes('HEAD')) changed.push('HEAD'); await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; - if (changed.length === 0) return { read, changed }; + if (changed.length === 0) return { read, state: 'untouched' }; // Not `git stash push`: the stash list is shared by every worktree of the checkout. const kept = `refs/teamai/kept/${lease.attempt}`; const added = await git(cwd, ['add', '-A']); const stashed = added.code === 0 ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'stash', 'create']) : added; const recorded = stashed.code === 0 ? await git(cwd, ['update-ref', kept, stashed.stdout.trim() || 'HEAD']) : stashed; // Without the ref, a reset would lose the changes. - if (recorded.code !== 0) return { read, changed, unrestored: { error: recorded.stderr.trim() || 'its changes could not be kept', dir: cwd } }; + if (recorded.code !== 0) return { read, state: 'unrestored', changed, reason: recorded.stderr.trim() || 'its changes could not be kept', dir: cwd }; // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. // A superproject reset leaves initialized submodules where the step moved them. const submodules = [['submodule', '--quiet', 'update', '--recursive', '--force'], ['submodule', '--quiet', 'foreach', '--recursive', 'git clean -q -fd']]; for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ...submodules]) { const restored = await git(cwd, args); - if (restored.code !== 0) return { read, changed, kept, unrestored: { error: restored.stderr.trim() || `git ${args.join(' ')} failed`, dir: cwd } }; + if (restored.code !== 0) return { read, state: 'unrestored', changed, kept, reason: restored.stderr.trim() || `git ${args.join(' ')} failed`, dir: cwd }; } - return { read, changed, kept }; + return { read, state: 'restored', changed, kept }; } /** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ -function readCopyFailure(left: ReadCopy, changed: string): WorkflowFailure { - const kept = left.kept ? ` Its changes are kept on ${left.kept}.` : ''; - return left.unrestored - ? { code: 'workspace_base', message: `${changed} Workspace ${left.read.branch} could NOT be put back at ${left.read.commit}: ${left.unrestored.error}.${kept} Put ${left.unrestored.dir} back on ${left.read.branch} at ${left.read.commit} before a later step runs there.` } - : { code: 'read_only_dirty', message: `${changed} Workspace ${left.read.branch} was put back at ${left.read.commit}.${kept}` }; +function readCopyFailure(left: Exclude, changed: string): WorkflowFailure { + const { branch, commit } = left.read; + switch (left.state) { + case 'restored': + return { code: 'read_only_dirty', message: `${changed} Workspace ${branch} was put back at ${commit}. Its changes are kept on ${left.kept}.` }; + case 'unrestored': + return { + code: 'workspace_base', + message: left.kept + ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes are kept on ${left.kept}. Put ${left.dir} back on ${branch} at ${commit} before a later step runs there.` + : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit}, and before a later step runs there.`, + }; + default: { + const unhandled: never = left; + return unhandled; + } + } } async function finishWorkspace(lease: Lease, push: boolean): Promise<{ changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; failure?: WorkflowFailure }> { @@ -597,8 +614,9 @@ async function commandReport(attemptResultPath: string, driver: DriverIdentity, } } const left = await leaveReadCopy(lease); - const dirty = left && (result.exitCode === 0 || left.unrestored) && left.changed.length > 0 - ? readCopyFailure(left, `Read-only command step changed repository paths: ${left.changed.join(', ')}.`) + const touched = left?.state === 'untouched' ? undefined : left; + const dirty = touched && (result.exitCode === 0 || touched.state === 'unrestored') + ? readCopyFailure(touched, `Read-only command step changed repository paths: ${touched.changed.join(', ')}.`) : undefined; const finished = lease.workspace && !left ? await finishWorkspace(lease, result.exitCode === 0) : {}; const failure = dirty ?? finished.failure; @@ -700,10 +718,11 @@ async function promptReport( }; } const left = await leaveReadCopy(lease); - const paths = left ? left.changed : before && !lease.workspace ? changedPaths(before, await snapshotWorktree(cwd)) : []; - const dirtyFailure = paths.length === 0 ? undefined : left - ? readCopyFailure(left, `Read-only prompt step changed repository paths: ${paths.join(', ')}.`) - : { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}` }; + const touched = left?.state === 'untouched' ? undefined : left; + const paths = !left && before && !lease.workspace ? changedPaths(before, await snapshotWorktree(cwd)) : []; + const dirtyFailure = touched + ? readCopyFailure(touched, `Read-only prompt step changed repository paths: ${touched.changed.join(', ')}.`) + : paths.length === 0 ? undefined : { code: 'read_only_dirty', message: `Read-only prompt step changed repository paths: ${paths.join(', ')}` }; const finished = lease.workspace && !left && !dirtyFailure ? await finishWorkspace(lease, result.exitCode === 0 && !result.failure) : {}; const failure = dirtyFailure ?? finished.failure ?? result.failure; const ran = result.started ? result : undefined; From 5c4ba8c710f4dac51e01d444215425971c1b682a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:10:32 +0200 Subject: [PATCH 12/41] fix(workflow): refuse later steps in a copy a reader could not put back After a failed restore the driver still leased later steps into that copy, so a continue writer could commit on a detached HEAD while reporting the named branch, or commit what the reader left. leaveReadCopy now writes a marker in the worktree's own git dir. stageLease refuses a workspace lease while the copy is not on refs/heads/ at the marker's commit with a clean status, and removes the marker once it is, so the person who puts the copy back only runs teamai workflow retry. --- src/__tests__/workflow-workspace.test.ts | 43 ++++++++++++++ src/workflow/local-driver.ts | 75 +++++++++++++++++++++--- src/workflow/worktree.ts | 2 +- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index b562d8678..ad6b17207 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2504,6 +2504,49 @@ steps: expect(test).not.toMatchObject({ failure: { message: expect.stringContaining('was put back') } }); }); + it('refuses a later writer in a copy a reader left off its branch, and runs it on the branch once the copy is back and the step is retried', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A copy that could not be put back blocks the next writer until a person puts it back. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'); const b=cp.execFileSync('git',['symbolic-ref','--short','HEAD']).toString().trim(); cp.execFileSync('git',['checkout','-q','--detach']); cp.execFileSync('git',['branch','-qD',b])" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`teamai workflow retry ${runId} change.polish`) }, + }); + const copy = worktreePath(workspace, branch); + expect(fs.existsSync(path.join(copy, 'polish.txt'))).toBe(false); + git(['checkout', '-q', '-b', branch], copy); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 2 } } }); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const again = await readRun(config, runId); + if (typeof again !== 'object') throw new Error('run missing'); + expect(again.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded', changes: { branch } }); + expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); + }); + it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index fbd537028..f4952df38 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -4,6 +4,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { createInterface } from 'node:readline/promises'; +import { z } from 'zod'; import { agentFileExtensionForTool, reverseFromClaude, @@ -42,7 +43,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, worktreePath, + applyBaseConfig, commitWork, ensureWorktree, git, gitPath, headCommit, hooksPath, mergeBranch, pushIfOrigin, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -366,7 +367,7 @@ function placeOf(lease: Lease, active: string): string { return lease.cwd ?? (lease.workspace ? worktreePath(active, lease.workspace.branch) : active); } -async function stageLease(active: string, lease: Lease): Promise { +async function stageLease(lease: Lease, { active, runId }: { active: string; runId: string }): Promise { if (lease.unreadableWorkspace) { return { code: 'workspace_base', message: `Step "${lease.step}" was leased a workspace this teamai cannot read. Update teamai.` }; } @@ -375,6 +376,14 @@ async function stageLease(active: string, lease: Lease): Promise { lease.overlay = undefined; } +const UnrestoredMarker = z.object({ branch: z.string(), commit: z.string(), step: z.string(), reason: z.string() }); +type UnrestoredMarker = z.infer; + +/** In the worktree's own git dir, so the marker never shows in its status. */ +const UNRESTORED_MARKER = 'teamai-unrestored'; + +/** Why `dir` is not on `branch` at `commit` with nothing changed, submodules included. Undefined when it is. */ +async function copyLeftOver(dir: string, branch: string, commit: string): Promise { + const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); + if (head !== `refs/heads/${branch}`) return head ? `HEAD is on ${head}` : 'HEAD is detached'; + const at = (await git(dir, ['rev-parse', 'HEAD'])).stdout.trim(); + if (at !== commit) return `HEAD is at ${at}`; + const status = await git(dir, ['status', '--porcelain', '--ignore-submodules=none']); + if (status.code !== 0) return status.stderr.trim() || 'git status failed'; + const lines = status.stdout.split('\n').filter((line) => line.trim() !== ''); + if (lines.length === 0) return undefined; + return `it has changes: ${lines.slice(0, 5).map((line) => line.trim()).join(', ')}${lines.length > 5 ? `, and ${lines.length - 5} more` : ''}`; +} + +/** + * A copy a reader could not put back, while it is still not back. Once it is, the marker is removed, so the person who + * put it back only retries the step. + */ +async function unrestoredCopy(dir: string): Promise< + { state: 'unrestored'; dir: string; marker: UnrestoredMarker; left: string } | { state: 'unreadable'; dir: string; marker: string } | undefined +> { + const file = await gitPath(dir, UNRESTORED_MARKER); + const text = file ? await readFileSafe(file) : null; + if (!file || text === null) return undefined; + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { state: 'unreadable', dir, marker: file }; + } + const parsed = UnrestoredMarker.safeParse(raw); + if (!parsed.success) return { state: 'unreadable', dir, marker: file }; + const left = await copyLeftOver(dir, parsed.data.branch, parsed.data.commit); + if (left) return { state: 'unrestored', dir, marker: parsed.data, left }; + await fs.rm(file, { force: true }); + return undefined; +} + /** How a reader left its copy. `kept` is the ref holding what the step changed; an unrestored copy lacks it when keeping failed. */ type ReadCopy = { read: StepRead } & ( | { state: 'untouched' } @@ -544,19 +596,26 @@ async function leaveReadCopy(lease: Lease): Promise { await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; if (changed.length === 0) return { read, state: 'untouched' }; + // Later steps in this copy are refused until it is put back, so nothing commits what was left here. + const unrestored = async (reason: string, kept?: string): Promise => { + const marker = await gitPath(cwd, UNRESTORED_MARKER); + const record: UnrestoredMarker = { branch: read.branch, commit: read.commit, step: lease.step, reason }; + if (marker) await writeJsonAtomic(marker, record); + return { read, state: 'unrestored', changed, reason, dir: cwd, ...(kept ? { kept } : {}) }; + }; // Not `git stash push`: the stash list is shared by every worktree of the checkout. const kept = `refs/teamai/kept/${lease.attempt}`; const added = await git(cwd, ['add', '-A']); const stashed = added.code === 0 ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'stash', 'create']) : added; const recorded = stashed.code === 0 ? await git(cwd, ['update-ref', kept, stashed.stdout.trim() || 'HEAD']) : stashed; // Without the ref, a reset would lose the changes. - if (recorded.code !== 0) return { read, state: 'unrestored', changed, reason: recorded.stderr.trim() || 'its changes could not be kept', dir: cwd }; + if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. // A superproject reset leaves initialized submodules where the step moved them. const submodules = [['submodule', '--quiet', 'update', '--recursive', '--force'], ['submodule', '--quiet', 'foreach', '--recursive', 'git clean -q -fd']]; for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ...submodules]) { const restored = await git(cwd, args); - if (restored.code !== 0) return { read, state: 'unrestored', changed, kept, reason: restored.stderr.trim() || `git ${args.join(' ')} failed`, dir: cwd }; + if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } return { read, state: 'restored', changed, kept }; } @@ -571,8 +630,8 @@ function readCopyFailure(left: Exclude, change return { code: 'workspace_base', message: left.kept - ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes are kept on ${left.kept}. Put ${left.dir} back on ${branch} at ${commit} before a later step runs there.` - : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit}, and before a later step runs there.`, + ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes are kept on ${left.kept}. Later steps there fail until ${left.dir} is back on ${branch} at ${commit} with nothing changed.` + : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit} with nothing changed. Later steps there fail until it is.`, }; default: { const unhandled: never = left; @@ -884,7 +943,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options continue; } held.splice(index, 1); - const failure = await stageLease(cwd, lease); + const failure = await stageLease(lease, { active: cwd, runId }); if (failure) { const failed = await failStaged(lease, failure); if (failed) return failed; @@ -905,7 +964,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options if (failed) return failed; continue; } - const failure = await stageLease(cwd, lease); + const failure = await stageLease(lease, { active: cwd, runId }); if (failure) { const failed = await failStaged(lease, failure); if (failed) return failed; diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index aaa9efdc4..893c1f74c 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -76,7 +76,7 @@ export function worktreePath(main: string, branch: string): string { } /** A path git keeps for this checkout. A linked worktree's `.git` is a file, so the path is not `/.git`. */ -async function gitPath(cwd: string, name: string): Promise { +export async function gitPath(cwd: string, name: string): Promise { const result = await git(cwd, ['rev-parse', '--git-path', name]); const printed = result.code === 0 ? result.stdout.trim() : ''; return printed === '' ? undefined : path.resolve(cwd, printed); From 89a18aed93fc31419e1d89b64bd532376ad0fc2b Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:12:01 +0200 Subject: [PATCH 13/41] fix(workflow): leave a submodule a reader edited untouched instead of destroying it git stash create in the superproject never saves uncommitted or untracked files inside a submodule, yet submodule update --force and a foreach git clean discarded them, while the failure said the changes were kept. Before any keep or restore, each initialized submodule is checked recursively. When one has such files, nothing is reset, and the reader fails unrestored naming them, which also blocks later steps in that copy. The foreach clean is gone. --- src/__tests__/workflow-workspace.test.ts | 42 ++++++++++++++++++++++-- src/workflow/local-driver.ts | 15 +++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index ad6b17207..66ebf5d31 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2550,7 +2550,7 @@ steps: it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build -description: A reader's commit and untracked file inside a submodule do not reach the branch. +description: A reader's commit inside a submodule does not reach the branch. steps: - id: change workspace: new @@ -2561,11 +2561,11 @@ steps: command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('impl.txt','yes')" - id: test needs: [implement] - command: node -e "const cp=require('child_process'),fs=require('fs'); fs.writeFileSync('vendor/inside.txt','reader'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd:'vendor'}); fs.writeFileSync('vendor/untracked.txt','x')" + command: node -e "const cp=require('child_process'),fs=require('fs'); fs.writeFileSync('vendor/inside.txt','reader'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd:'vendor'})" - id: polish needs: [implement] workspace: continue - command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inside.txt','utf8')!=='one\\\\n' || fs.existsSync('vendor/untracked.txt')) process.exit(2); fs.writeFileSync('polish.txt','yes')" + command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inside.txt','utf8')!=='one\\\\n') process.exit(2); fs.writeFileSync('polish.txt','yes')" `); const recorded = addSubmodule(workspace); @@ -2580,6 +2580,42 @@ steps: expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); }); + it('leaves a submodule a reader edited untouched, fails it unrestored, and refuses the next step in that copy', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Uncommitted and untracked files inside a submodule are not kept, so they stay. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const fs=require('fs'); fs.writeFileSync('vendor/inside.txt','reader'); fs.writeFileSync('vendor/untracked.txt','x')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + addSubmodule(workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('vendor') } }); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ failure: { message: expect.stringContaining('could NOT be put back') } }); + expect(fs.readFileSync(path.join(copy, 'vendor', 'inside.txt'), 'utf8')).toBe('reader'); + expect(fs.existsSync(path.join(copy, 'vendor', 'untracked.txt'))).toBe(true); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`teamai workflow retry ${runId} change.polish`) } }); + expect(() => git(['show', `${branch}:polish.txt`], workspace)).toThrow(); + }); + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index f4952df38..f7b2a8881 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -574,6 +574,14 @@ async function unrestoredCopy(dir: string): Promise< return undefined; } +/** Initialized submodules, at any depth, with uncommitted or untracked files. */ +async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { error: string }> { + // Each submodule's own status ignores its nested submodules: --recursive visits those itself. + const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --ignore-submodules=all)" || echo "$displaypath"']); + if (listed.code !== 0) return { error: listed.stderr.trim() || 'git submodule foreach failed' }; + return { paths: listed.stdout.split('\n').filter((line) => line !== '') }; +} + /** How a reader left its copy. `kept` is the ref holding what the step changed; an unrestored copy lacks it when keeping failed. */ type ReadCopy = { read: StepRead } & ( | { state: 'untouched' } @@ -603,6 +611,10 @@ async function leaveReadCopy(lease: Lease): Promise { if (marker) await writeJsonAtomic(marker, record); return { read, state: 'unrestored', changed, reason, dir: cwd, ...(kept ? { kept } : {}) }; }; + // The keep below records a submodule only as its commit, so a submodule update would destroy its files. + const dirty = await dirtySubmodules(cwd); + if ('error' in dirty) return unrestored(dirty.error); + if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); // Not `git stash push`: the stash list is shared by every worktree of the checkout. const kept = `refs/teamai/kept/${lease.attempt}`; const added = await git(cwd, ['add', '-A']); @@ -612,8 +624,7 @@ async function leaveReadCopy(lease: Lease): Promise { if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. // A superproject reset leaves initialized submodules where the step moved them. - const submodules = [['submodule', '--quiet', 'update', '--recursive', '--force'], ['submodule', '--quiet', 'foreach', '--recursive', 'git clean -q -fd']]; - for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ...submodules]) { + for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--recursive', '--force']]) { const restored = await git(cwd, args); if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } From 2baf740804e30dc5c55e60fca72803f79340d186 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:13:05 +0200 Subject: [PATCH 14/41] fix(workflow): check a read copy is really back before saying it was put back git clean -fd skips a repository the reader created inside the copy, every restore command still exits 0, and the next writer's git add -A committed it as a gitlink while the failure said the copy was put back. After the restore, the copy now counts as put back only when HEAD is refs/heads/ at the commit it read and git status, submodules included, is empty. Otherwise the reader fails unrestored naming what is left, and the nested repository stays. --- src/__tests__/workflow-workspace.test.ts | 34 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 4 ++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 66ebf5d31..116e83d80 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2547,6 +2547,40 @@ steps: expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); }); + it('fails a reader that created a git repository in its copy unrestored, keeps that repository, and refuses the next writer there', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A nested repository survives git clean, so the copy is not put back. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'); cp.execFileSync('git',['init','-q','nested']); require('fs').writeFileSync('nested/a.txt','x'); cp.execFileSync('git',['add','a.txt'],{cwd:'nested'}); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','fixture'],{cwd:'nested'})" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'change', 1); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringMatching(/could NOT be put back.*nested/) }, + }); + expect(fs.existsSync(path.join(worktreePath(workspace, branch), 'nested', 'a.txt'))).toBe(true); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); + expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); + }); + it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index f7b2a8881..285d02c19 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -628,7 +628,9 @@ async function leaveReadCopy(lease: Lease): Promise { const restored = await git(cwd, args); if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } - return { read, state: 'restored', changed, kept }; + // Exit codes are not enough: git clean skips a repository the step created, and a later git add -A commits it. + const left = await copyLeftOver(cwd, read.branch, read.commit); + return left ? unrestored(left, kept) : { read, state: 'restored', changed, kept }; } /** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ From 6d45ecf529576b4d18ec3703e211bc52ce883bcb Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:15:27 +0200 Subject: [PATCH 15/41] fix(workflow): restore a reader's submodules with the checkout procedure submodule update without --checkout follows a configured update=merge, rebase or none, so the reader's submodule commit survived the restore and a later writer committed that gitlink. The restore now passes --checkout. --force is dropped: a dirty submodule never reaches this point, and a clean one checks out without it. --- src/__tests__/workflow-workspace.test.ts | 34 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 116e83d80..89f2268ea 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2614,6 +2614,40 @@ steps: expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); }); + it('puts a submodule whose update mode is rebase back at the recorded commit after a reader commits in it', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A configured submodule update mode does not keep the reader's commit. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'),fs=require('fs'); fs.writeFileSync('vendor/inside.txt','reader'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd:'vendor'})" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + const recorded = addSubmodule(workspace); + git(['config', '-f', '.gitmodules', 'submodule.vendor.update', 'rebase'], workspace); + git(['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qam', 'rebase'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('was put back') } }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); + const branch = branchName(runId, 'change', 1); + expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); + }); + it('leaves a submodule a reader edited untouched, fails it unrestored, and refuses the next step in that copy', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 285d02c19..442a4e7fc 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -624,7 +624,7 @@ async function leaveReadCopy(lease: Lease): Promise { if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. // A superproject reset leaves initialized submodules where the step moved them. - for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--recursive', '--force']]) { + for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { const restored = await git(cwd, args); if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } From 3b644d38ced562ba030b37a6c7f5799024cca527 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:16:35 +0200 Subject: [PATCH 16/41] fix(workflow): compare a reader's HEAD with the full branch ref symbolic-ref --short prints heads/ when a tag has the branch's name, so a reader that changed nothing looked like it had moved HEAD and failed read_only_dirty. --- src/__tests__/workflow-workspace.test.ts | 22 ++++++++++++++++++++++ src/workflow/local-driver.ts | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 89f2268ea..edd2679c1 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2425,6 +2425,28 @@ steps: expect(() => git(['show', `${branch}:junk.txt`], workspace)).toThrow(); }); + it('does not fail a reader that changed nothing when a tag has its branch name', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A tag named like the copy's branch is not a move off the branch. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['tag',require('child_process').execFileSync('git',['symbolic-ref','--short','HEAD']).toString().trim()]); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "0" +`); + + const done = await runLocalDriver(config, runId, workspace); + + expect(done, JSON.stringify(done)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + }); + it('keeps what a reader changed on a ref named in its failure before putting the copy back', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 442a4e7fc..10804961a 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -599,7 +599,8 @@ async function leaveReadCopy(lease: Lease): Promise { if (!cwd || !before?.head || workspace?.mode !== 'read-only') return undefined; const changed = changedPaths(before, await snapshotWorktree(cwd)); // A detach or a switch to another branch at the same commit changes no file, but a later step would commit off the branch. - const onBranch = (await git(cwd, ['symbolic-ref', '--quiet', '--short', 'HEAD'])).stdout.trim() === workspace.branch; + // The full ref: --short prints heads/ when a tag has the same name. + const onBranch = (await git(cwd, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim() === `refs/heads/${workspace.branch}`; if (!onBranch && !changed.includes('HEAD')) changed.push('HEAD'); await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; From b40a993d7982c81157df4cdd1529094458ce3a47 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:18:13 +0200 Subject: [PATCH 17/41] fix(workflow): name the kept ref when a read-only command that changed its copy exits non-zero The changes of a failing reader still went to refs/teamai/kept/ and were removed from the copy, but its report was only the exit status, so nothing said where they went. The command's exit status and output snippet stay first, followed by the same put-back or could NOT be put back sentence a passing reader gets. --- src/__tests__/workflow-workspace.test.ts | 4 +++- src/workflow/local-driver.ts | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index edd2679c1..d8eeec8c4 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2417,7 +2417,9 @@ steps: if (typeof run !== 'object') throw new Error('run missing'); const test = run.steps.find((step) => step.id === 'change.test'); const polish = run.steps.find((step) => step.id === 'change.polish'); - expect(test).toMatchObject({ status: 'failed', ...(failure ? { failure } : { message: 'command exited with status 1' }) }); + const kept = `refs/teamai/kept/${test && 'attempt' in test ? test.attempt : 'none'}`; + expect(test).toMatchObject({ status: 'failed', ...(failure ? { failure } : { message: expect.stringMatching(/^command exited with status 1\. Read-only command step changed repository paths: junk\.txt\. .*was put back/) }) }); + expect(test).toMatchObject({ message: expect.stringContaining(kept) }); expect(polish).toMatchObject({ status: 'succeeded' }); const branch = branchName(runId, 'change', 1); expect(test && 'read' in test ? test.read?.commit : undefined).toBe(polish && 'changes' in polish ? polish.changes?.from : 'none'); diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 10804961a..149b53939 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -688,14 +688,16 @@ async function commandReport(attemptResultPath: string, driver: DriverIdentity, } const left = await leaveReadCopy(lease); const touched = left?.state === 'untouched' ? undefined : left; - const dirty = touched && (result.exitCode === 0 || touched.state === 'unrestored') - ? readCopyFailure(touched, `Read-only command step changed repository paths: ${touched.changed.join(', ')}.`) - : undefined; + const exited = result.exitCode === 0 + ? undefined + : snippet === '' ? `command exited with status ${result.exitCode}` : `command exited with status ${result.exitCode}: ${snippet}`; + // A failing command keeps its own failure, and still says where the changes it made went. + const copy = touched && readCopyFailure(touched, `${exited ? `${exited}. ` : ''}Read-only command step changed repository paths: ${touched.changed.join(', ')}.`); + const dirty = copy && (!exited || copy.code === 'workspace_base') ? copy : undefined; const finished = lease.workspace && !left ? await finishWorkspace(lease, result.exitCode === 0) : {}; const failure = dirty ?? finished.failure; const failed = result.exitCode !== 0 || failure; - const message = failure?.message - ?? (result.exitCode !== 0 ? (snippet === '' ? `command exited with status ${result.exitCode}` : `command exited with status ${result.exitCode}: ${snippet}`) : undefined); + const message = failure?.message ?? copy?.message ?? exited; return { attempt: lease.attempt, revision: lease.revision, From 379608fe049e18b8eae4a83605a858dbf81d54b1 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:20:28 +0200 Subject: [PATCH 18/41] fix(workflow): refuse an existing workflow worktree that is not on its branch ensureWorktree returned an existing worktree without checking HEAD, so after a driver stopped before it put a read copy back, the next writer committed on a detached HEAD and reported the named branch. An existing worktree whose HEAD is not refs/heads/ now fails workspace_base, naming the directory, where HEAD is, and the branch to check out. --- src/__tests__/workflow-workspace.test.ts | 35 ++++++++++++++++++++++++ src/workflow/worktree.ts | 10 ++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index d8eeec8c4..1e1dbd3bc 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2605,6 +2605,41 @@ steps: expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); }); + it('refuses a writer whose existing copy is on a detached HEAD, naming the branch to put it back on', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A copy left detached, as by a driver stopped before it put the copy back, is not reused. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('../../../go')) process.exit(1); fs.writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + await runLocalDriver(config, runId, workspace); + const copy = worktreePath(workspace, branch); + git(['checkout', '-q', '--detach'], copy); + fs.writeFileSync(path.join(workspace, 'go'), ''); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 2 } } }); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringMatching(new RegExp(`${copy.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} is detached at .*not on branch ${branch}`)) }, + }); + expect(() => git(['show', `${branch}:polish.txt`], workspace)).toThrow(); + expect(fs.existsSync(path.join(copy, 'polish.txt'))).toBe(false); + }); + it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 893c1f74c..2cc2b5ee7 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -107,7 +107,10 @@ export async function headCommit(cwd: string): Promise<{ ref?: string; commit: s return ref ? { ref, commit } : { commit }; } -/** Creates the worktree when it is missing. A branch that already exists is checked out as it is. */ +/** + * Creates the worktree when it is missing. A branch that already exists is checked out as it is. An existing worktree + * off its branch is refused, so a step never commits on whatever HEAD was left. + */ export async function ensureWorktree(main: string, branch: string, start: string): Promise<{ dir: string } | { error: string }> { const dir = worktreePath(main, branch); const hooks = hooksPath(main); @@ -117,6 +120,11 @@ export async function ensureWorktree(main: string, branch: string, start: string const known = listed.code === 0 && listed.stdout.split('\n').some((line) => line.startsWith('worktree ') && path.resolve(line.slice('worktree '.length)) === path.resolve(dir)); const existing = known || await git(dir, ['rev-parse', '--is-inside-work-tree']).then((result) => result.stdout.trim() === 'true').catch(() => false); if (existing) { + const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); + if (head !== `refs/heads/${branch}`) { + const on = head ? `on ${head}` : `detached at ${(await git(dir, ['rev-parse', 'HEAD'])).stdout.trim()}`; + return { error: `Workspace ${dir} is ${on}, not on branch ${branch}. Check out ${branch} in ${dir}, then retry the step.` }; + } await worktreeHooks(main, dir, hooks); return { dir }; } From 85ffcb9a8f53b091d71cb6dd0769ae1dc0f760de Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:22:00 +0200 Subject: [PATCH 19/41] fix(workflow): point a symlinked agent path at the fix that can work The base_config message said to commit the symlinked parent as a directory on the branch and retry. When the symlink comes from the run base, every run branch starts with it and no step can fix it. That case now says to commit it as a directory on the base branch and start a new run. The branch case names the branch and the worktree to commit in. --- src/__tests__/workflow-workspace.test.ts | 27 ++++++++++++++++++++++++ src/workflow/worktree.ts | 8 ++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 1e1dbd3bc..4ffd41640 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -731,9 +731,36 @@ steps: const run = await readRun(config, runId); const check = typeof run === 'object' ? run.steps.find((step) => step.id === 'work.check') : undefined; expect(check).toMatchObject({ status: 'failed', failure: { code: 'base_config', message: expect.stringContaining('.github') } }); + const branch = branchName(runId, 'work', 1); + expect(check).toMatchObject({ failure: { message: expect.stringContaining(`Commit .github as a directory on branch ${branch} in ${worktreePath(workspace, branch)}, then retry the step.`) } }); expect(fs.readFileSync(path.join(outside, 'skills', 'keep.txt'), 'utf8')).toBe('keep'); }); + it('fails a workspace step with base_config, telling the user to fix the base branch and start a new run, when the run base has .github as a symlink', async () => { + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-outside-')); + dirs.push(outside); + const { config, workspace } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A symlink in the run base cannot be fixed by a step. +steps: + - id: work + workspace: new + command: node -e "require('fs').writeFileSync('ok.txt','yes')" +`); + fs.symlinkSync(outside, path.join(workspace, '.github')); + execFileSync('git', ['add', '.github'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'link'], { cwd: workspace }); + const started = await startRun(config, 'build'); + if ('error' in started) throw new Error(started.error.message); + const runId = String(started.data.run); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + const work = typeof run === 'object' ? run.steps.find((step) => step.id === 'work') : undefined; + expect(work).toMatchObject({ status: 'failed', failure: { code: 'base_config', message: expect.stringContaining('Commit .github as a directory on the base branch, then start a new run.') } }); + }); + it('keeps .github workflows from the branch and resets only its agent paths', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 2cc2b5ee7..035bbd05c 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -168,7 +168,13 @@ export async function applyBaseConfig( await git(dir, ['config', '--worktree', 'core.hooksPath', hooks]); const linked = await symlinkedParent(dir); if (linked) { - return { ok: false, message: `Agent configuration ${linked.relative} is under the symlink ${linked.parent} on this branch, so it was not reset. Commit ${linked.parent} as a directory on the branch and retry the step.` }; + // Every run branch starts from the base, so no step can fix a symlink the base has. + const inBase = (await git(dir, ['ls-tree', commit, '--', linked.parent])).stdout.startsWith('120000 '); + if (inBase) { + return { ok: false, message: `Agent configuration ${linked.relative} is under the symlink ${linked.parent} in the run's base commit ${commit}, so it was not reset. Commit ${linked.parent} as a directory on the base branch, then start a new run.` }; + } + const branch = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim().replace(/^refs\/heads\//, ''); + return { ok: false, message: `Agent configuration ${linked.relative} is under the symlink ${linked.parent} on branch ${branch}, so it was not reset. Commit ${linked.parent} as a directory on branch ${branch} in ${dir}, then retry the step.` }; } const saved = await fs.mkdtemp(path.join(os.tmpdir(), 'teamai-agent-')); const conflicts = (await git(dir, ['ls-files', '-u', '-z', '--', ...AGENT_PATHS])).stdout; From 4da1dfdb7c424ce6ba2dcc2a2b536bb7c10925d9 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:22:28 +0200 Subject: [PATCH 20/41] docs(workflow): say an unrestored read copy blocks later steps until it is put back --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 51b42bc90..4b483bf23 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, `git stash create`; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and the same for initialized submodules, so a later writer never commits a reader's edits. A git failure there fails `workspace_base` and says the copy was not put back. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, `git stash create`; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when an initialized submodule has uncommitted or untracked files, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --ignore-submodules=none` is empty. Otherwise the reader fails `workspace_base`, and a `teamai-unrestored` marker in the worktree's git dir makes `stageLease` refuse later leases there until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 8f6f25031..df29e0b1e 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -139,7 +139,7 @@ Use `workflow respond ` for a waiting HITL request, `workfl A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. -Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, the worktree and its submodules are reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. If git cannot put it back, the step fails with `workspace_base`. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. +Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, the worktree and its submodules are reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. A submodule with uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`, and later steps there fail until you put it back on its branch at that commit with nothing changed and run `teamai workflow retry `. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. `workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 563f07b06..f177df515 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -139,7 +139,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom 没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 -在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。git 无法恢复工作区时，步骤以 `workspace_base` 失败。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 +在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败；之后在该工作区的步骤都会失败，直到你把它放回它的分支、停在那个提交且没有任何改动，再运行 `teamai workflow retry `。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 `workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 From 315b583d7934c5865311ef8a66e95977a943f11c Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:31:58 +0200 Subject: [PATCH 21/41] fix(workflow): keep a reader's submodule commit on its kept ref --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-workspace.test.ts | 5 ++++- src/workflow/local-driver.ts | 9 ++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 4b483bf23..438434800 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, `git stash create`; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when an initialized submodule has uncommitted or untracked files, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --ignore-submodules=none` is empty. Otherwise the reader fails `workspace_base`, and a `teamai-unrestored` marker in the worktree's git dir makes `stageLease` refuse later leases there until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD, which also keeps a submodule's commit; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when an initialized submodule has uncommitted or untracked files, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --ignore-submodules=none` is empty. Otherwise the reader fails `workspace_base`, and a `teamai-unrestored` marker in the worktree's git dir makes `stageLease` refuse later leases there until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 4ffd41640..08f68b121 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2693,11 +2693,14 @@ steps: const run = await readRun(config, runId); if (typeof run !== 'object') throw new Error('run missing'); - expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('vendor') } }); + const test = run.steps.find((step) => step.id === 'change.test'); + expect(test).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('vendor') } }); expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); const branch = branchName(runId, 'change', 1); expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); + const kept = `refs/teamai/kept/${test && 'attempt' in test ? test.attempt : 'none'}`; + expect(git(['rev-parse', `${kept}:vendor`], workspace).trim()).not.toBe(recorded); }); it('puts a submodule whose update mode is rebase back at the recorded commit after a reader commits in it', async () => { diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 149b53939..26f7b27b9 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -616,11 +616,14 @@ async function leaveReadCopy(lease: Lease): Promise { const dirty = await dirtySubmodules(cwd); if ('error' in dirty) return unrestored(dirty.error); if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); - // Not `git stash push`: the stash list is shared by every worktree of the checkout. + // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. const kept = `refs/teamai/kept/${lease.attempt}`; const added = await git(cwd, ['add', '-A']); - const stashed = added.code === 0 ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'stash', 'create']) : added; - const recorded = stashed.code === 0 ? await git(cwd, ['update-ref', kept, stashed.stdout.trim() || 'HEAD']) : stashed; + const tree = added.code === 0 ? await git(cwd, ['write-tree']) : added; + const committed = tree.code === 0 + ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit-tree', tree.stdout.trim(), '-p', 'HEAD', '-m', `teamai: kept ${lease.step}`]) + : tree; + const recorded = committed.code === 0 ? await git(cwd, ['update-ref', kept, committed.stdout.trim()]) : committed; // Without the ref, a reset would lose the changes. if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. From 41165763a66282d58613d8364937ce64937a2ad5 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:49:24 +0200 Subject: [PATCH 22/41] refactor(workflow): move putting a read copy back into worktree.ts local-driver.ts had crossed 1,000 lines. copyLeftOver, dirtySubmodules, the unrestored marker and the keep-and-restore sequence are git-level handling of a worktree and take no lease, so they move to worktree.ts as putBackCopy and unrestoredCopy. leaveReadCopy keeps the snapshot, overlay and lease wiring. No behaviour change. --- src/workflow/local-driver.ts | 86 ++-------------------------------- src/workflow/worktree.ts | 91 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 83 deletions(-) diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 26f7b27b9..0c4c9ff54 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -4,7 +4,6 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { createInterface } from 'node:readline/promises'; -import { z } from 'zod'; import { agentFileExtensionForTool, reverseFromClaude, @@ -43,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, ensureWorktree, git, gitPath, headCommit, hooksPath, mergeBranch, pushIfOrigin, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, worktreePath, + applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -531,57 +530,6 @@ async function restoreLeaseOverlay(lease: Lease): Promise { lease.overlay = undefined; } -const UnrestoredMarker = z.object({ branch: z.string(), commit: z.string(), step: z.string(), reason: z.string() }); -type UnrestoredMarker = z.infer; - -/** In the worktree's own git dir, so the marker never shows in its status. */ -const UNRESTORED_MARKER = 'teamai-unrestored'; - -/** Why `dir` is not on `branch` at `commit` with nothing changed, submodules included. Undefined when it is. */ -async function copyLeftOver(dir: string, branch: string, commit: string): Promise { - const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); - if (head !== `refs/heads/${branch}`) return head ? `HEAD is on ${head}` : 'HEAD is detached'; - const at = (await git(dir, ['rev-parse', 'HEAD'])).stdout.trim(); - if (at !== commit) return `HEAD is at ${at}`; - const status = await git(dir, ['status', '--porcelain', '--ignore-submodules=none']); - if (status.code !== 0) return status.stderr.trim() || 'git status failed'; - const lines = status.stdout.split('\n').filter((line) => line.trim() !== ''); - if (lines.length === 0) return undefined; - return `it has changes: ${lines.slice(0, 5).map((line) => line.trim()).join(', ')}${lines.length > 5 ? `, and ${lines.length - 5} more` : ''}`; -} - -/** - * A copy a reader could not put back, while it is still not back. Once it is, the marker is removed, so the person who - * put it back only retries the step. - */ -async function unrestoredCopy(dir: string): Promise< - { state: 'unrestored'; dir: string; marker: UnrestoredMarker; left: string } | { state: 'unreadable'; dir: string; marker: string } | undefined -> { - const file = await gitPath(dir, UNRESTORED_MARKER); - const text = file ? await readFileSafe(file) : null; - if (!file || text === null) return undefined; - let raw: unknown; - try { - raw = JSON.parse(text); - } catch { - return { state: 'unreadable', dir, marker: file }; - } - const parsed = UnrestoredMarker.safeParse(raw); - if (!parsed.success) return { state: 'unreadable', dir, marker: file }; - const left = await copyLeftOver(dir, parsed.data.branch, parsed.data.commit); - if (left) return { state: 'unrestored', dir, marker: parsed.data, left }; - await fs.rm(file, { force: true }); - return undefined; -} - -/** Initialized submodules, at any depth, with uncommitted or untracked files. */ -async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { error: string }> { - // Each submodule's own status ignores its nested submodules: --recursive visits those itself. - const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --ignore-submodules=all)" || echo "$displaypath"']); - if (listed.code !== 0) return { error: listed.stderr.trim() || 'git submodule foreach failed' }; - return { paths: listed.stdout.split('\n').filter((line) => line !== '') }; -} - /** How a reader left its copy. `kept` is the ref holding what the step changed; an unrestored copy lacks it when keeping failed. */ type ReadCopy = { read: StepRead } & ( | { state: 'untouched' } @@ -605,36 +553,8 @@ async function leaveReadCopy(lease: Lease): Promise { await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; if (changed.length === 0) return { read, state: 'untouched' }; - // Later steps in this copy are refused until it is put back, so nothing commits what was left here. - const unrestored = async (reason: string, kept?: string): Promise => { - const marker = await gitPath(cwd, UNRESTORED_MARKER); - const record: UnrestoredMarker = { branch: read.branch, commit: read.commit, step: lease.step, reason }; - if (marker) await writeJsonAtomic(marker, record); - return { read, state: 'unrestored', changed, reason, dir: cwd, ...(kept ? { kept } : {}) }; - }; - // The keep below records a submodule only as its commit, so a submodule update would destroy its files. - const dirty = await dirtySubmodules(cwd); - if ('error' in dirty) return unrestored(dirty.error); - if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); - // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. - const kept = `refs/teamai/kept/${lease.attempt}`; - const added = await git(cwd, ['add', '-A']); - const tree = added.code === 0 ? await git(cwd, ['write-tree']) : added; - const committed = tree.code === 0 - ? await git(cwd, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit-tree', tree.stdout.trim(), '-p', 'HEAD', '-m', `teamai: kept ${lease.step}`]) - : tree; - const recorded = committed.code === 0 ? await git(cwd, ['update-ref', kept, committed.stdout.trim()]) : committed; - // Without the ref, a reset would lose the changes. - if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); - // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. - // A superproject reset leaves initialized submodules where the step moved them. - for (const args of [['checkout', '-q', '-f', workspace.branch], ['reset', '-q', '--hard', before.head], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { - const restored = await git(cwd, args); - if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); - } - // Exit codes are not enough: git clean skips a repository the step created, and a later git add -A commits it. - const left = await copyLeftOver(cwd, read.branch, read.commit); - return left ? unrestored(left, kept) : { read, state: 'restored', changed, kept }; + const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, attempt: lease.attempt, step: lease.step }); + return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; } /** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 035bbd05c..18527d4b5 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -8,7 +8,9 @@ import { execFile, spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { z } from 'zod'; import { TeamaiConfigBaseSchema } from '../types.js'; +import { readFileSafe, writeJsonAtomic } from '../utils/fs.js'; /** Agent configuration a workspace step reads from the run's base rather than from its branch. */ export const AGENT_PATHS: readonly string[] = agentConfigPaths(); @@ -518,3 +520,92 @@ export async function removeWorktree(main: string, dir: string): Promise { await git(main, ['worktree', 'remove', '--force', dir]); await git(main, ['worktree', 'prune']); } + +const UnrestoredMarker = z.object({ branch: z.string(), commit: z.string(), step: z.string(), reason: z.string() }); +export type UnrestoredMarker = z.infer; + +/** In the worktree's own git dir, so the marker never shows in its status. */ +const UNRESTORED_MARKER = 'teamai-unrestored'; + +/** Why `dir` is not on `branch` at `commit` with nothing changed, submodules included. Undefined when it is. */ +async function copyLeftOver(dir: string, branch: string, commit: string): Promise { + const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); + if (head !== `refs/heads/${branch}`) return head ? `HEAD is on ${head}` : 'HEAD is detached'; + const at = (await git(dir, ['rev-parse', 'HEAD'])).stdout.trim(); + if (at !== commit) return `HEAD is at ${at}`; + const status = await git(dir, ['status', '--porcelain', '--ignore-submodules=none']); + if (status.code !== 0) return status.stderr.trim() || 'git status failed'; + const lines = status.stdout.split('\n').filter((line) => line.trim() !== ''); + if (lines.length === 0) return undefined; + return `it has changes: ${lines.slice(0, 5).map((line) => line.trim()).join(', ')}${lines.length > 5 ? `, and ${lines.length - 5} more` : ''}`; +} + +/** + * A copy a reader could not put back, while it is still not back. Once it is, the marker is removed, so the person who + * put it back only retries the step. + */ +export async function unrestoredCopy(dir: string): Promise< + { state: 'unrestored'; dir: string; marker: UnrestoredMarker; left: string } | { state: 'unreadable'; dir: string; marker: string } | undefined +> { + const file = await gitPath(dir, UNRESTORED_MARKER); + const text = file ? await readFileSafe(file) : null; + if (!file || text === null) return undefined; + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { state: 'unreadable', dir, marker: file }; + } + const parsed = UnrestoredMarker.safeParse(raw); + if (!parsed.success) return { state: 'unreadable', dir, marker: file }; + const left = await copyLeftOver(dir, parsed.data.branch, parsed.data.commit); + if (left) return { state: 'unrestored', dir, marker: parsed.data, left }; + await fs.rm(file, { force: true }); + return undefined; +} + +/** Initialized submodules, at any depth, with uncommitted or untracked files. */ +async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { error: string }> { + // Each submodule's own status ignores its nested submodules: --recursive visits those itself. + const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --ignore-submodules=all)" || echo "$displaypath"']); + if (listed.code !== 0) return { error: listed.stderr.trim() || 'git submodule foreach failed' }; + return { paths: listed.stdout.split('\n').filter((line) => line !== '') }; +} + +/** + * Resets a copy a reader changed to `commit` on `branch`, after keeping what it changed on `refs/teamai/kept/`. + * A copy it cannot put back is marked, so later steps there are refused until it is. + */ +export async function putBackCopy({ dir, branch, commit, attempt, step }: { dir: string; branch: string; commit: string; attempt: string; step: string }): Promise< + { state: 'restored'; kept: string } | { state: 'unrestored'; reason: string; kept?: string } +> { + const unrestored = async (reason: string, kept?: string): Promise<{ state: 'unrestored'; reason: string; kept?: string }> => { + const marker = await gitPath(dir, UNRESTORED_MARKER); + const record: UnrestoredMarker = { branch, commit, step, reason }; + if (marker) await writeJsonAtomic(marker, record); + return { state: 'unrestored', reason, ...(kept ? { kept } : {}) }; + }; + // The keep below records a submodule only as its commit, so a submodule update would destroy its files. + const dirty = await dirtySubmodules(dir); + if ('error' in dirty) return unrestored(dirty.error); + if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); + // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. + const kept = `refs/teamai/kept/${attempt}`; + const added = await git(dir, ['add', '-A']); + const tree = added.code === 0 ? await git(dir, ['write-tree']) : added; + const committed = tree.code === 0 + ? await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit-tree', tree.stdout.trim(), '-p', 'HEAD', '-m', `teamai: kept ${step}`]) + : tree; + const recorded = committed.code === 0 ? await git(dir, ['update-ref', kept, committed.stdout.trim()]) : committed; + // Without the ref, a reset would lose the changes. + if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); + // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. + // A superproject reset leaves initialized submodules where the step moved them. + for (const args of [['checkout', '-q', '-f', branch], ['reset', '-q', '--hard', commit], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { + const restored = await git(dir, args); + if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); + } + // Exit codes are not enough: git clean skips a repository the step created, and a later git add -A commits it. + const left = await copyLeftOver(dir, branch, commit); + return left ? unrestored(left, kept) : { state: 'restored', kept }; +} From 6f3202dfa8d416abe1685858b8bc1b201c87394d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:51:25 +0200 Subject: [PATCH 23/41] fix(workflow): put a read copy back to the status it had when the reader started The postcondition required an empty status, so a submodule an earlier writer left dirty could never pass, and it read the user's status config, so status.showUntrackedFiles=no hid a repository the reader left. The copy's status is now captured at staging, before the overlay, and put back means that status again, read with --untracked-files=all. --- src/__tests__/workflow-workspace.test.ts | 34 ++++++++++++++++++++++ src/workflow/local-driver.ts | 15 +++++++--- src/workflow/worktree.ts | 36 +++++++++++++++--------- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 08f68b121..4b89bf8a6 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2632,6 +2632,40 @@ steps: expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); }); + it('fails a reader that created a git repository in its copy unrestored when the repository hides untracked files from status', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: status.showUntrackedFiles=no does not hide a repository the reader left. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'); cp.execFileSync('git',['init','-q','nested']); require('fs').writeFileSync('nested/a.txt','x'); cp.execFileSync('git',['add','a.txt'],{cwd:'nested'}); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','fixture'],{cwd:'nested'})" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + git(['config', 'status.showUntrackedFiles', 'no'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'change', 1); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringMatching(/could NOT be put back.*nested/) }, + }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); + expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); + }); + it('refuses a writer whose existing copy is on a detached HEAD, naming the branch to put it back on', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 0c4c9ff54..18df41b7d 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -42,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, + applyBaseConfig, commitWork, copyStatus, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -163,6 +163,8 @@ interface Lease { /** Set once the step has a directory to run in. */ cwd?: string; before?: WorktreeSnapshot; + /** A read copy's status before the base overlay, which putting the copy back returns it to. */ + found?: string[]; /** Branch agent files saved while the base overlay is in place. */ overlay?: string; /** HEAD before this step merges or commits, so the reported range includes both. */ @@ -386,6 +388,11 @@ async function stageLease(lease: Lease, { active, runId }: { active: string; run const opened = await ensureWorktree(active, lease.workspace.branch, lease.workspace.start.commit); if ('error' in opened) return { code: 'workspace_base', message: opened.error }; lease.cwd = opened.dir; + if (lease.workspace.mode === 'read-only') { + const found = await copyStatus(opened.dir); + if ('error' in found) return { code: 'workspace_base', message: `Step "${lease.step}" could not read workspace ${lease.workspace.branch}: ${found.error}` }; + lease.found = found.lines; + } lease.rangeFrom = (await headCommit(opened.dir))?.commit; let conflict: { branch: string; files: string[] } | undefined; const remaining: string[] = []; @@ -543,8 +550,8 @@ type ReadCopy = { read: StepRead } & ( * Undefined for any other lease. */ async function leaveReadCopy(lease: Lease): Promise { - const { cwd, before, workspace } = lease; - if (!cwd || !before?.head || workspace?.mode !== 'read-only') return undefined; + const { cwd, before, found, workspace } = lease; + if (!cwd || !before?.head || !found || workspace?.mode !== 'read-only') return undefined; const changed = changedPaths(before, await snapshotWorktree(cwd)); // A detach or a switch to another branch at the same commit changes no file, but a later step would commit off the branch. // The full ref: --short prints heads/ when a tag has the same name. @@ -553,7 +560,7 @@ async function leaveReadCopy(lease: Lease): Promise { await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; if (changed.length === 0) return { read, state: 'untouched' }; - const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, attempt: lease.attempt, step: lease.step }); + const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, found, attempt: lease.attempt, step: lease.step }); return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; } diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 18527d4b5..072aa6372 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -521,21 +521,30 @@ export async function removeWorktree(main: string, dir: string): Promise { await git(main, ['worktree', 'prune']); } -const UnrestoredMarker = z.object({ branch: z.string(), commit: z.string(), step: z.string(), reason: z.string() }); +const UnrestoredMarker = z.object({ branch: z.string(), commit: z.string(), step: z.string(), reason: z.string(), found: z.array(z.string()) }); export type UnrestoredMarker = z.infer; /** In the worktree's own git dir, so the marker never shows in its status. */ const UNRESTORED_MARKER = 'teamai-unrestored'; -/** Why `dir` is not on `branch` at `commit` with nothing changed, submodules included. Undefined when it is. */ -async function copyLeftOver(dir: string, branch: string, commit: string): Promise { +/** A copy's status, with every untracked file and submodule change whatever the user's status config hides. */ +export async function copyStatus(dir: string): Promise<{ lines: string[] } | { error: string }> { + const status = await git(dir, ['status', '--porcelain', '--untracked-files=all', '--ignore-submodules=none']); + if (status.code !== 0) return { error: status.stderr.trim() || 'git status failed' }; + return { lines: status.stdout.split('\n').filter((line) => line.trim() !== '') }; +} + +/** Why `dir` is not on `branch` at `commit` with the status `found`, submodules included. Undefined when it is. */ +async function copyLeftOver(dir: string, { branch, commit, found }: { branch: string; commit: string; found: string[] }): Promise { const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); if (head !== `refs/heads/${branch}`) return head ? `HEAD is on ${head}` : 'HEAD is detached'; const at = (await git(dir, ['rev-parse', 'HEAD'])).stdout.trim(); if (at !== commit) return `HEAD is at ${at}`; - const status = await git(dir, ['status', '--porcelain', '--ignore-submodules=none']); - if (status.code !== 0) return status.stderr.trim() || 'git status failed'; - const lines = status.stdout.split('\n').filter((line) => line.trim() !== ''); + const status = await copyStatus(dir); + if ('error' in status) return status.error; + const was = new Set(found); + const now = new Set(status.lines); + const lines = [...status.lines.filter((line) => !was.has(line)), ...found.filter((line) => !now.has(line))]; if (lines.length === 0) return undefined; return `it has changes: ${lines.slice(0, 5).map((line) => line.trim()).join(', ')}${lines.length > 5 ? `, and ${lines.length - 5} more` : ''}`; } @@ -558,7 +567,7 @@ export async function unrestoredCopy(dir: string): Promise< } const parsed = UnrestoredMarker.safeParse(raw); if (!parsed.success) return { state: 'unreadable', dir, marker: file }; - const left = await copyLeftOver(dir, parsed.data.branch, parsed.data.commit); + const left = await copyLeftOver(dir, parsed.data); if (left) return { state: 'unrestored', dir, marker: parsed.data, left }; await fs.rm(file, { force: true }); return undefined; @@ -567,21 +576,22 @@ export async function unrestoredCopy(dir: string): Promise< /** Initialized submodules, at any depth, with uncommitted or untracked files. */ async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { error: string }> { // Each submodule's own status ignores its nested submodules: --recursive visits those itself. - const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --ignore-submodules=all)" || echo "$displaypath"']); + const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --untracked-files=all --ignore-submodules=all)" || echo "$displaypath"']); if (listed.code !== 0) return { error: listed.stderr.trim() || 'git submodule foreach failed' }; return { paths: listed.stdout.split('\n').filter((line) => line !== '') }; } /** - * Resets a copy a reader changed to `commit` on `branch`, after keeping what it changed on `refs/teamai/kept/`. - * A copy it cannot put back is marked, so later steps there are refused until it is. + * Puts a copy a reader changed back on `branch` at `commit` with the status `found` it had when the reader started, after + * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back is marked, so later steps there are + * refused until it is. */ -export async function putBackCopy({ dir, branch, commit, attempt, step }: { dir: string; branch: string; commit: string; attempt: string; step: string }): Promise< +export async function putBackCopy({ dir, branch, commit, found, attempt, step }: { dir: string; branch: string; commit: string; found: string[]; attempt: string; step: string }): Promise< { state: 'restored'; kept: string } | { state: 'unrestored'; reason: string; kept?: string } > { const unrestored = async (reason: string, kept?: string): Promise<{ state: 'unrestored'; reason: string; kept?: string }> => { const marker = await gitPath(dir, UNRESTORED_MARKER); - const record: UnrestoredMarker = { branch, commit, step, reason }; + const record: UnrestoredMarker = { branch, commit, step, reason, found }; if (marker) await writeJsonAtomic(marker, record); return { state: 'unrestored', reason, ...(kept ? { kept } : {}) }; }; @@ -606,6 +616,6 @@ export async function putBackCopy({ dir, branch, commit, attempt, step }: { dir: if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } // Exit codes are not enough: git clean skips a repository the step created, and a later git add -A commits it. - const left = await copyLeftOver(dir, branch, commit); + const left = await copyLeftOver(dir, { branch, commit, found }); return left ? unrestored(left, kept) : { state: 'restored', kept }; } From fb93a665f22d7afc2d950ff29a775c526f9e8a46 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:54:30 +0200 Subject: [PATCH 24/41] fix(workflow): block a read copy's put-back only on repositories the reader changed Every dirty submodule blocked the restore, so a submodule an earlier writer left dirty failed a reader that never touched it. Only repositories among the reader's changed paths are checked now. Inside each, status ignores only dirty nested submodules, so a commit made in vendor/inner alone shows as a change in vendor and the copy is left untouched instead of being reset. A submodule with no checkout left counts as unkept. The reset and checkout no longer recurse into submodules under submodule.recurse=true. --- src/__tests__/workflow-workspace.test.ts | 79 +++++++++++++++++++++++- src/workflow/local-driver.ts | 2 +- src/workflow/worktree.ts | 51 ++++++++++++--- 3 files changed, 120 insertions(+), 12 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 4b89bf8a6..26e1e6c3a 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -112,12 +112,13 @@ async function teamWorkspace(yaml: string): Promise<{ config: LocalConfig; works } /** Commits a submodule `vendor` holding `inside.txt` (`one`) to the workspace, and returns the commit it records. */ -function addSubmodule(workspace: string): string { +function addSubmodule(workspace: string, inner?: string): string { const source = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); dirs.push(source); execFileSync('git', ['init', '-q'], { cwd: source }); fs.writeFileSync(path.join(source, 'inside.txt'), 'one\n'); execFileSync('git', ['add', 'inside.txt'], { cwd: source }); + if (inner) execFileSync('git', ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', inner, 'inner'], { cwd: source }); execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: source }); execFileSync('git', ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', source, 'vendor'], { cwd: workspace }); execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'vendor'], { cwd: workspace }); @@ -2807,6 +2808,82 @@ steps: expect(() => git(['show', `${branch}:polish.txt`], workspace)).toThrow(); }); + it('puts the copy back around a submodule an earlier writer left dirty when the reader changed only a superproject file', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A reader is not blamed for a submodule it did not touch. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('vendor/inside.txt','writer'); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').writeFileSync('report.txt','reader')" + - id: polish + needs: [test] + workspace: continue + command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inside.txt','utf8')!=='writer') process.exit(2); fs.writeFileSync('polish.txt','yes')" +`); + addSubmodule(workspace); + git(['config', 'submodule.recurse', 'true'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const test = run.steps.find((step) => step.id === 'change.test'); + const kept = `refs/teamai/kept/${test && 'attempt' in test ? test.attempt : 'none'}`; + expect(test).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining(kept) } }); + expect(git(['show', `${kept}:report.txt`], workspace)).toBe('reader'); + }); + + it('leaves the copy untouched when a reader commits only in a nested submodule, and says its change was NOT saved', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A commit in vendor/inner shows only as an uncommitted gitlink in vendor, which the kept ref cannot hold. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init','--recursive']); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'),fs=require('fs'); fs.writeFileSync('vendor/inner/inside.txt','reader'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd:'vendor/inner'})" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + const inner = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); + dirs.push(inner); + execFileSync('git', ['init', '-q'], { cwd: inner }); + fs.writeFileSync(path.join(inner, 'inside.txt'), 'one\n'); + execFileSync('git', ['add', 'inside.txt'], { cwd: inner }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: inner }); + const original = git(['rev-parse', 'HEAD'], inner).trim(); + addSubmodule(workspace, inner); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`Its changes were NOT saved and are still in ${copy}`) }, + }); + expect(git(['rev-parse', 'HEAD'], path.join(copy, 'vendor', 'inner')).trim()).not.toBe(original); + expect(fs.readFileSync(path.join(copy, 'vendor', 'inner', 'inside.txt'), 'utf8')).toBe('reader'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); + }); + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 18df41b7d..14dbb3831 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -560,7 +560,7 @@ async function leaveReadCopy(lease: Lease): Promise { await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; if (changed.length === 0) return { read, state: 'untouched' }; - const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, found, attempt: lease.attempt, step: lease.step }); + const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, found, changed, attempt: lease.attempt, step: lease.step }); return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; } diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 072aa6372..6e45bd9a9 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -573,12 +573,39 @@ export async function unrestoredCopy(dir: string): Promise< return undefined; } -/** Initialized submodules, at any depth, with uncommitted or untracked files. */ -async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { error: string }> { - // Each submodule's own status ignores its nested submodules: --recursive visits those itself. - const listed = await git(dir, ['submodule', '--quiet', 'foreach', '--recursive', 'test -z "$(git status --porcelain --untracked-files=all --ignore-submodules=all)" || echo "$displaypath"']); - if (listed.code !== 0) return { error: listed.stderr.trim() || 'git submodule foreach failed' }; - return { paths: listed.stdout.split('\n').filter((line) => line !== '') }; +/** + * Repositories among `paths`, submodules and ones the step created, whose changes the kept commit cannot hold: uncommitted + * or untracked files, or a nested submodule's commit its parent has not recorded. A submodule with no checkout left counts + * too, since its files may have gone with it. + */ +async function unkeptRepositories(dir: string, { paths, commit }: { paths: string[]; commit: string }): Promise<{ paths: string[] } | { error: string }> { + const listed = [await git(dir, ['ls-tree', '-r', '-z', commit]), await git(dir, ['ls-files', '-s', '-z'])]; + const failed = listed.find((entries) => entries.code !== 0); + if (failed) return { error: failed.stderr.trim() || 'the submodules could not be listed' }; + const gitlinks = new Set(listed.flatMap((entries) => entries.stdout.split('\0')) + .filter((entry) => entry.startsWith('160000 ')).map((entry) => entry.slice(entry.indexOf('\t') + 1))); + // Dirty, not all: a nested submodule's new commit shows only as a change in its parent. + const status = ['status', '--porcelain', '--untracked-files=all', '--ignore-submodules=dirty']; + const unkept: string[] = []; + for (const relative of paths) { + // Status lists a repository the step created as its directory with a trailing slash. + const name = relative.replace(/\/$/, ''); + if (!gitlinks.has(name) && name === relative) continue; + const repo = path.join(dir, name); + const top = await git(repo, ['rev-parse', '--show-prefix']); + if (top.code !== 0 || top.stdout.trim() !== '') { + if (gitlinks.has(name)) unkept.push(name); + continue; + } + for (const args of [status, ['submodule', '--quiet', 'foreach', '--recursive', `test -z "$(git ${status.join(' ')})" || echo "$displaypath"`]]) { + const listed = await git(repo, args); + if (listed.code !== 0) return { error: listed.stderr.trim() || `git ${args[0]} failed in ${name}` }; + if (listed.stdout.trim() === '') continue; + unkept.push(name); + break; + } + } + return { paths: unkept }; } /** @@ -586,7 +613,9 @@ async function dirtySubmodules(dir: string): Promise<{ paths: string[] } | { err * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back is marked, so later steps there are * refused until it is. */ -export async function putBackCopy({ dir, branch, commit, found, attempt, step }: { dir: string; branch: string; commit: string; found: string[]; attempt: string; step: string }): Promise< +export async function putBackCopy({ dir, branch, commit, found, changed, attempt, step }: { + dir: string; branch: string; commit: string; found: string[]; changed: string[]; attempt: string; step: string; +}): Promise< { state: 'restored'; kept: string } | { state: 'unrestored'; reason: string; kept?: string } > { const unrestored = async (reason: string, kept?: string): Promise<{ state: 'unrestored'; reason: string; kept?: string }> => { @@ -596,7 +625,7 @@ export async function putBackCopy({ dir, branch, commit, found, attempt, step }: return { state: 'unrestored', reason, ...(kept ? { kept } : {}) }; }; // The keep below records a submodule only as its commit, so a submodule update would destroy its files. - const dirty = await dirtySubmodules(dir); + const dirty = await unkeptRepositories(dir, { paths: changed, commit }); if ('error' in dirty) return unrestored(dirty.error); if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. @@ -610,8 +639,10 @@ export async function putBackCopy({ dir, branch, commit, found, attempt, step }: // Without the ref, a reset would lose the changes. if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. - // A superproject reset leaves initialized submodules where the step moved them. - for (const args of [['checkout', '-q', '-f', branch], ['reset', '-q', '--hard', commit], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { + // A superproject reset leaves initialized submodules where the step moved them. It must not recurse into a submodule an + // earlier step left dirty. + const noRecurse = ['-c', 'submodule.recurse=false']; + for (const args of [[...noRecurse, 'checkout', '-q', '-f', branch], [...noRecurse, 'reset', '-q', '--hard', commit], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { const restored = await git(dir, args); if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } From 55ddd8b1ad94512877d9e528e6a003f3d21b49d2 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:56:35 +0200 Subject: [PATCH 25/41] fix(workflow): keep a reader's submodule commits reachable after the copy is removed A gitlink on the kept ref does not keep its commit reachable, and removing the copy deletes the submodule's own store. Each submodule commit the kept tree records and the read commit does not, including a repository the reader created, is pushed into the workspace repository under refs/teamai/kept-modules// before any reset. A failed push is a failed keep: nothing is reset and the message says the changes were NOT saved. --- src/__tests__/workflow-workspace.test.ts | 12 ++++- src/workflow/local-driver.ts | 18 +++++--- src/workflow/worktree.ts | 59 ++++++++++++++++++++---- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 26e1e6c3a..3018ca16c 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2631,6 +2631,9 @@ steps: expect(fs.existsSync(path.join(worktreePath(workspace, branch), 'nested', 'a.txt'))).toBe(true); expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); + const test = run.steps.find((step) => step.id === 'change.test'); + const attempt = test && 'attempt' in test ? test.attempt : 'none'; + expect(git(['show', `refs/teamai/kept-modules/${attempt}/nested:a.txt`], workspace)).toBe('x'); }); it('fails a reader that created a git repository in its copy unrestored when the repository hides untracked files from status', async () => { @@ -2734,8 +2737,13 @@ steps: const branch = branchName(runId, 'change', 1); expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); expect(git(['rev-parse', `${branch}:vendor`], workspace).trim()).toBe(recorded); - const kept = `refs/teamai/kept/${test && 'attempt' in test ? test.attempt : 'none'}`; - expect(git(['rev-parse', `${kept}:vendor`], workspace).trim()).not.toBe(recorded); + const attempt = test && 'attempt' in test ? test.attempt : 'none'; + const reader = git(['rev-parse', `refs/teamai/kept/${attempt}:vendor`], workspace).trim(); + expect(reader).not.toBe(recorded); + expect(fs.existsSync(worktreePath(workspace, branch))).toBe(false); + expect(git(['cat-file', '-t', reader], workspace).trim()).toBe('commit'); + expect(git(['rev-parse', `refs/teamai/kept-modules/${attempt}/vendor`], workspace).trim()).toBe(reader); + expect(test).toMatchObject({ failure: { message: expect.stringContaining(`refs/teamai/kept-modules/${attempt}/`) } }); }); it('puts a submodule whose update mode is rebase back at the recorded commit after a reader commits in it', async () => { diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 14dbb3831..20ceb0a50 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -42,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, copyStatus, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, + applyBaseConfig, commitWork, copyStatus, type KeptChanges, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -537,11 +537,11 @@ async function restoreLeaseOverlay(lease: Lease): Promise { lease.overlay = undefined; } -/** How a reader left its copy. `kept` is the ref holding what the step changed; an unrestored copy lacks it when keeping failed. */ +/** How a reader left its copy. `kept` holds what the step changed; an unrestored copy lacks it when keeping failed. */ type ReadCopy = { read: StepRead } & ( | { state: 'untouched' } - | { state: 'restored'; changed: string[]; kept: string } - | { state: 'unrestored'; changed: string[]; reason: string; dir: string; kept?: string } + | { state: 'restored'; changed: string[]; kept: KeptChanges } + | { state: 'unrestored'; changed: string[]; reason: string; dir: string; kept?: KeptChanges } ); /** @@ -564,17 +564,23 @@ async function leaveReadCopy(lease: Lease): Promise { return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; } +function keptSentence(kept: KeptChanges): string { + return kept.modules + ? `Its changes are kept on ${kept.ref}, and the submodule commits it made under ${kept.modules}.` + : `Its changes are kept on ${kept.ref}.`; +} + /** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ function readCopyFailure(left: Exclude, changed: string): WorkflowFailure { const { branch, commit } = left.read; switch (left.state) { case 'restored': - return { code: 'read_only_dirty', message: `${changed} Workspace ${branch} was put back at ${commit}. Its changes are kept on ${left.kept}.` }; + return { code: 'read_only_dirty', message: `${changed} Workspace ${branch} was put back at ${commit}. ${keptSentence(left.kept)}` }; case 'unrestored': return { code: 'workspace_base', message: left.kept - ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes are kept on ${left.kept}. Later steps there fail until ${left.dir} is back on ${branch} at ${commit} with nothing changed.` + ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. ${keptSentence(left.kept)} Later steps there fail until ${left.dir} is back on ${branch} at ${commit} with nothing changed.` : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit} with nothing changed. Later steps there fail until it is.`, }; default: { diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 6e45bd9a9..3f04c900f 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -573,17 +573,51 @@ export async function unrestoredCopy(dir: string): Promise< return undefined; } +function gitlinkLines(listed: string): string[] { + return listed.split('\0').filter((line) => line.startsWith('160000 ')); +} + +/** Each submodule path `commit` records, with the commit it records there. */ +async function treeGitlinks(dir: string, commit: string): Promise | { error: string }> { + const listed = await git(dir, ['ls-tree', '-r', '-z', commit]); + if (listed.code !== 0) return { error: listed.stderr.trim() || `git ls-tree ${commit} failed` }; + return new Map(gitlinkLines(listed.stdout).map((line) => { + const tab = line.indexOf('\t'); + return [line.slice(tab + 1), line.slice(0, tab).split(' ')[2] ?? '']; + })); +} + +/** + * Copies each submodule commit `to` records and `from` does not into this repository's own store. A gitlink does not keep + * its commit reachable, and removing the copy deletes the submodule's store with it. + */ +async function keepModuleCommits(dir: string, { from, to, attempt }: { from: string; to: string; attempt: string }): Promise<{ modules?: string } | { error: string }> { + const before = await treeGitlinks(dir, from); + const after = await treeGitlinks(dir, to); + if ('error' in before) return before; + if ('error' in after) return after; + const moved = [...after].filter(([relative, commit]) => before.get(relative) !== commit); + if (moved.length === 0) return {}; + const common = await git(dir, ['rev-parse', '--path-format=absolute', '--git-common-dir']); + if (common.code !== 0) return { error: common.stderr.trim() || 'the git directory could not be found' }; + const modules = `refs/teamai/kept-modules/${attempt}/`; + for (const [relative, commit] of moved) { + const pushed = await git(path.join(dir, relative), ['push', '-q', common.stdout.trim(), `${commit}:${modules}${relative}`]); + if (pushed.code !== 0) return { error: pushed.stderr.trim() || `the commit in ${relative} could not be kept` }; + } + return { modules }; +} + /** * Repositories among `paths`, submodules and ones the step created, whose changes the kept commit cannot hold: uncommitted * or untracked files, or a nested submodule's commit its parent has not recorded. A submodule with no checkout left counts * too, since its files may have gone with it. */ async function unkeptRepositories(dir: string, { paths, commit }: { paths: string[]; commit: string }): Promise<{ paths: string[] } | { error: string }> { - const listed = [await git(dir, ['ls-tree', '-r', '-z', commit]), await git(dir, ['ls-files', '-s', '-z'])]; - const failed = listed.find((entries) => entries.code !== 0); - if (failed) return { error: failed.stderr.trim() || 'the submodules could not be listed' }; - const gitlinks = new Set(listed.flatMap((entries) => entries.stdout.split('\0')) - .filter((entry) => entry.startsWith('160000 ')).map((entry) => entry.slice(entry.indexOf('\t') + 1))); + const recorded = await treeGitlinks(dir, commit); + const staged = await git(dir, ['ls-files', '-s', '-z']); + if ('error' in recorded || staged.code !== 0) return { error: 'error' in recorded ? recorded.error : staged.stderr.trim() || 'git ls-files failed' }; + const gitlinks = new Set([...recorded.keys(), ...gitlinkLines(staged.stdout).map((line) => line.slice(line.indexOf('\t') + 1))]); // Dirty, not all: a nested submodule's new commit shows only as a change in its parent. const status = ['status', '--porcelain', '--untracked-files=all', '--ignore-submodules=dirty']; const unkept: string[] = []; @@ -608,6 +642,9 @@ async function unkeptRepositories(dir: string, { paths, commit }: { paths: strin return { paths: unkept }; } +/** Where a reader's changes were kept: the whole tree on `ref`, and each submodule commit it made under `modules`. */ +export type KeptChanges = { ref: string; modules?: string }; + /** * Puts a copy a reader changed back on `branch` at `commit` with the status `found` it had when the reader started, after * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back is marked, so later steps there are @@ -616,9 +653,9 @@ async function unkeptRepositories(dir: string, { paths, commit }: { paths: strin export async function putBackCopy({ dir, branch, commit, found, changed, attempt, step }: { dir: string; branch: string; commit: string; found: string[]; changed: string[]; attempt: string; step: string; }): Promise< - { state: 'restored'; kept: string } | { state: 'unrestored'; reason: string; kept?: string } + { state: 'restored'; kept: KeptChanges } | { state: 'unrestored'; reason: string; kept?: KeptChanges } > { - const unrestored = async (reason: string, kept?: string): Promise<{ state: 'unrestored'; reason: string; kept?: string }> => { + const unrestored = async (reason: string, kept?: KeptChanges): Promise<{ state: 'unrestored'; reason: string; kept?: KeptChanges }> => { const marker = await gitPath(dir, UNRESTORED_MARKER); const record: UnrestoredMarker = { branch, commit, step, reason, found }; if (marker) await writeJsonAtomic(marker, record); @@ -629,15 +666,19 @@ export async function putBackCopy({ dir, branch, commit, found, changed, attempt if ('error' in dirty) return unrestored(dirty.error); if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. - const kept = `refs/teamai/kept/${attempt}`; + const ref = `refs/teamai/kept/${attempt}`; const added = await git(dir, ['add', '-A']); const tree = added.code === 0 ? await git(dir, ['write-tree']) : added; const committed = tree.code === 0 ? await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit-tree', tree.stdout.trim(), '-p', 'HEAD', '-m', `teamai: kept ${step}`]) : tree; - const recorded = committed.code === 0 ? await git(dir, ['update-ref', kept, committed.stdout.trim()]) : committed; + if (committed.code !== 0) return unrestored(committed.stderr.trim() || 'its changes could not be kept'); + const modules = await keepModuleCommits(dir, { from: commit, to: committed.stdout.trim(), attempt }); + if ('error' in modules) return unrestored(modules.error); + const recorded = await git(dir, ['update-ref', ref, committed.stdout.trim()]); // Without the ref, a reset would lose the changes. if (recorded.code !== 0) return unrestored(recorded.stderr.trim() || 'its changes could not be kept'); + const kept: KeptChanges = { ref, ...modules }; // Back on the copy's branch first: a reset on a branch the step switched to would leave the copy elsewhere. // A superproject reset leaves initialized submodules where the step moved them. It must not recurse into a submodule an // earlier step left dirty. From 8f65ab6e6c13693814b2f24c9eea3cc01f5a6ce8 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 03:58:40 +0200 Subject: [PATCH 26/41] fix(workflow): mark a read copy when its reader starts, so a killed driver leaves it refused The marker was written only after a failed put-back, so a driver killed while the reader ran left no marker, and on resume a later writer committed the reader's files. Staging a read-only lease now writes the marker, with the copy's status, before the overlay, and fails workspace_base if it cannot. A copy put back or left untouched clears it; an unrestored one keeps it with the reason. A resumed copy that is back as it was found clears it by itself. --- src/__tests__/workflow-workspace.test.ts | 53 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 13 +++--- src/workflow/worktree.ts | 34 +++++++++++++-- 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 3018ca16c..b34a81acf 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2670,6 +2670,59 @@ steps: expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); }); + it('refuses a writer in a copy whose reader never finished, and clears that by itself once the copy is back', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: The marker staging writes outlives a driver killed mid-reader. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const m=require('child_process').execFileSync('git',['rev-parse','--git-path','teamai-unrestored']).toString().trim(); require('fs').copyFileSync(m,'../../../marker')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('../../../go')) process.exit(1); fs.writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + await runLocalDriver(config, runId, workspace); + const marker = path.resolve(copy, git(['rev-parse', '--git-path', 'teamai-unrestored'], copy).trim()); + expect(fs.existsSync(marker)).toBe(false); + const staged = fs.readFileSync(path.join(workspace, 'marker'), 'utf8'); + expect(JSON.parse(staged)).toMatchObject({ branch, step: 'change.test', reason: 'the reader did not finish', found: [] }); + // What a driver killed while the reader ran leaves: the marker as staging wrote it, and the reader's file. + fs.writeFileSync(marker, staged); + fs.writeFileSync(path.join(copy, 'junk.txt'), 'reader'); + fs.writeFileSync(path.join(workspace, 'go'), ''); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 2 } } }); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('junk.txt') }, + }); + expect(() => git(['show', `${branch}:junk.txt`], workspace)).toThrow(); + fs.rmSync(path.join(copy, 'junk.txt')); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 3 } } }); + + await runLocalDriver(config, runId, workspace); + + const again = await readRun(config, runId); + if (typeof again !== 'object') throw new Error('run missing'); + expect(again.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded', changes: { branch } }); + expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); + expect(() => git(['show', `${branch}:junk.txt`], workspace)).toThrow(); + }); + it('refuses a writer whose existing copy is on a detached HEAD, naming the branch to put it back on', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 20ceb0a50..b0952c516 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -42,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, copyStatus, type KeptChanges, ensureWorktree, git, headCommit, hooksPath, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, + applyBaseConfig, clearReadCopyMark, commitWork, ensureWorktree, git, headCommit, hooksPath, type KeptChanges, markReadCopy, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -389,9 +389,9 @@ async function stageLease(lease: Lease, { active, runId }: { active: string; run if ('error' in opened) return { code: 'workspace_base', message: opened.error }; lease.cwd = opened.dir; if (lease.workspace.mode === 'read-only') { - const found = await copyStatus(opened.dir); - if ('error' in found) return { code: 'workspace_base', message: `Step "${lease.step}" could not read workspace ${lease.workspace.branch}: ${found.error}` }; - lease.found = found.lines; + const marked = await markReadCopy(opened.dir, { branch: lease.workspace.branch, step: lease.step }); + if ('error' in marked) return { code: 'workspace_base', message: `Step "${lease.step}" cannot mark workspace ${opened.dir} as in use by a reader: ${marked.error}` }; + lease.found = marked.found; } lease.rangeFrom = (await headCommit(opened.dir))?.commit; let conflict: { branch: string; files: string[] } | undefined; @@ -559,7 +559,10 @@ async function leaveReadCopy(lease: Lease): Promise { if (!onBranch && !changed.includes('HEAD')) changed.push('HEAD'); await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; - if (changed.length === 0) return { read, state: 'untouched' }; + if (changed.length === 0) { + await clearReadCopyMark(cwd); + return { read, state: 'untouched' }; + } const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, found, changed, attempt: lease.attempt, step: lease.step }); return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; } diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 3f04c900f..fb4874390 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -528,7 +528,7 @@ export type UnrestoredMarker = z.infer; const UNRESTORED_MARKER = 'teamai-unrestored'; /** A copy's status, with every untracked file and submodule change whatever the user's status config hides. */ -export async function copyStatus(dir: string): Promise<{ lines: string[] } | { error: string }> { +async function copyStatus(dir: string): Promise<{ lines: string[] } | { error: string }> { const status = await git(dir, ['status', '--porcelain', '--untracked-files=all', '--ignore-submodules=none']); if (status.code !== 0) return { error: status.stderr.trim() || 'git status failed' }; return { lines: status.stdout.split('\n').filter((line) => line.trim() !== '') }; @@ -549,6 +549,30 @@ async function copyLeftOver(dir: string, { branch, commit, found }: { branch: st return `it has changes: ${lines.slice(0, 5).map((line) => line.trim()).join(', ')}${lines.length > 5 ? `, and ${lines.length - 5} more` : ''}`; } +/** + * Marks `dir` as left by a reader before the reader changes anything, with the status it has now. Putting the copy back + * removes the mark, so a driver killed before that leaves the copy refused to later steps until it is back. + */ +export async function markReadCopy(dir: string, { branch, step }: { branch: string; step: string }): Promise<{ found: string[] } | { error: string }> { + const status = await copyStatus(dir); + if ('error' in status) return status; + const head = await git(dir, ['rev-parse', 'HEAD']); + const marker = await gitPath(dir, UNRESTORED_MARKER); + if (head.code !== 0 || !marker) return { error: head.stderr.trim() || 'its git directory could not be found' }; + const record: UnrestoredMarker = { branch, commit: head.stdout.trim(), step, reason: 'the reader did not finish', found: status.lines }; + try { + await writeJsonAtomic(marker, record); + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } + return { found: status.lines }; +} + +export async function clearReadCopyMark(dir: string): Promise { + const marker = await gitPath(dir, UNRESTORED_MARKER); + if (marker) await fs.rm(marker, { force: true }); +} + /** * A copy a reader could not put back, while it is still not back. Once it is, the marker is removed, so the person who * put it back only retries the step. @@ -647,8 +671,8 @@ export type KeptChanges = { ref: string; modules?: string }; /** * Puts a copy a reader changed back on `branch` at `commit` with the status `found` it had when the reader started, after - * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back is marked, so later steps there are - * refused until it is. + * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back keeps its mark with the reason, so + * later steps there are refused until it is. */ export async function putBackCopy({ dir, branch, commit, found, changed, attempt, step }: { dir: string; branch: string; commit: string; found: string[]; changed: string[]; attempt: string; step: string; @@ -689,5 +713,7 @@ export async function putBackCopy({ dir, branch, commit, found, changed, attempt } // Exit codes are not enough: git clean skips a repository the step created, and a later git add -A commits it. const left = await copyLeftOver(dir, { branch, commit, found }); - return left ? unrestored(left, kept) : { state: 'restored', kept }; + if (left) return unrestored(left, kept); + await clearReadCopyMark(dir); + return { state: 'restored', kept }; } From f8bda44851174c2e0aa0a76014a061a5c34f3c2c Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:01:06 +0200 Subject: [PATCH 27/41] fix(workflow): recreate a deleted workflow copy, and refuse one that is not a worktree A copy whose directory was deleted stays registered, so worktree add refused its path; it is now pruned and created again. A plain directory at the copy path was told to check out the branch, which would switch the checkout around it; it is now refused as not a workflow worktree. A marker that exists but cannot be read now blocks the copy instead of counting as absent. --- src/__tests__/workflow-workspace.test.ts | 98 ++++++++++++++++++++++++ src/workflow/worktree.ts | 23 ++++-- 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index b34a81acf..548380c50 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2758,6 +2758,104 @@ steps: expect(fs.existsSync(path.join(copy, 'polish.txt'))).toBe(false); }); + it('creates the copy again when its directory was deleted', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A deleted copy is created again. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('../../../go')) process.exit(1); fs.writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + await runLocalDriver(config, runId, workspace); + fs.rmSync(worktreePath(workspace, branch), { recursive: true, force: true }); + fs.writeFileSync(path.join(workspace, 'go'), ''); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 2 } } }); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded', changes: { branch } }); + expect(git(['show', `${branch}:polish.txt`], workspace)).toBe('yes'); + }); + + it('refuses a directory at the copy path that is not a workflow worktree, without telling the user to check out a branch there', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A plain directory where the copy goes is not used. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('../../../go')) process.exit(1); fs.writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + fs.mkdirSync(copy, { recursive: true }); + const active = git(['symbolic-ref', 'HEAD'], workspace).trim(); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.implement')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`${copy} is not a workflow worktree`) }, + }); + expect(run.steps.find((step) => step.id === 'change.implement')).not.toMatchObject({ failure: { message: expect.stringContaining('Check out') } }); + expect(git(['symbolic-ref', 'HEAD'], workspace).trim()).toBe(active); + }); + + it('refuses a writer in a copy whose marker cannot be read', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: An unreadable marker fails closed. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('fs').writeFileSync('impl.txt','yes')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (!fs.existsSync('../../../go')) process.exit(1); fs.writeFileSync('polish.txt','yes')" +`); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + await runLocalDriver(config, runId, workspace); + fs.mkdirSync(path.resolve(copy, git(['rev-parse', '--git-path', 'teamai-unrestored'], copy).trim())); + fs.writeFileSync(path.join(workspace, 'go'), ''); + expect(await retryStep(config, runId, 'change.polish')).toMatchObject({ data: { retried: { pass: 2 } } }); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('has a marker teamai cannot read') }, + }); + expect(() => git(['show', `${branch}:polish.txt`], workspace)).toThrow(); + }); + it('puts a submodule a reader changed back at the commit the copy records, so a later step does not commit it', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index fb4874390..cc7cfefe4 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -10,7 +10,7 @@ import os from 'node:os'; import path from 'node:path'; import { z } from 'zod'; import { TeamaiConfigBaseSchema } from '../types.js'; -import { readFileSafe, writeJsonAtomic } from '../utils/fs.js'; +import { writeJsonAtomic } from '../utils/fs.js'; /** Agent configuration a workspace step reads from the run's base rather than from its branch. */ export const AGENT_PATHS: readonly string[] = agentConfigPaths(); @@ -119,9 +119,18 @@ export async function ensureWorktree(main: string, branch: string, start: string await fs.mkdir(hooks, { recursive: true }); await hideTeamaiWorktrees(main); const listed = await git(main, ['worktree', 'list', '--porcelain']); - const known = listed.code === 0 && listed.stdout.split('\n').some((line) => line.startsWith('worktree ') && path.resolve(line.slice('worktree '.length)) === path.resolve(dir)); - const existing = known || await git(dir, ['rev-parse', '--is-inside-work-tree']).then((result) => result.stdout.trim() === 'true').catch(() => false); + // Git lists the resolved path, which differs from `dir` under a symlinked temp or home directory. + const real = path.join(await fs.realpath(path.dirname(dir)).catch(() => path.dirname(dir)), path.basename(dir)); + const known = listed.code === 0 && listed.stdout.split('\n').some((line) => line.startsWith('worktree ') && [path.resolve(dir), real].includes(path.resolve(line.slice('worktree '.length)))); + const present = await fs.stat(dir).then((stats) => stats.isDirectory(), () => false); + // A deleted worktree stays registered, and `worktree add` refuses its path until it is pruned. + if (known && !present) await git(main, ['worktree', 'prune']); + const existing = present && (known || await git(dir, ['rev-parse', '--is-inside-work-tree']).then((result) => result.stdout.trim() === 'true').catch(() => false)); if (existing) { + const prefix = await git(dir, ['rev-parse', '--show-prefix']); + if (prefix.code !== 0 || prefix.stdout.trim() !== '') { + return { error: `${dir} is not a workflow worktree, so no step runs there. Move it out of the way, then retry the step.` }; + } const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); if (head !== `refs/heads/${branch}`) { const on = head ? `on ${head}` : `detached at ${(await git(dir, ['rev-parse', 'HEAD'])).stdout.trim()}`; @@ -581,12 +590,12 @@ export async function unrestoredCopy(dir: string): Promise< { state: 'unrestored'; dir: string; marker: UnrestoredMarker; left: string } | { state: 'unreadable'; dir: string; marker: string } | undefined > { const file = await gitPath(dir, UNRESTORED_MARKER); - const text = file ? await readFileSafe(file) : null; - if (!file || text === null) return undefined; + if (!file) return undefined; let raw: unknown; try { - raw = JSON.parse(text); - } catch { + raw = JSON.parse(await fs.readFile(file, 'utf8')); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return undefined; return { state: 'unreadable', dir, marker: file }; } const parsed = UnrestoredMarker.safeParse(raw); From f62f164b1a05277b1a8faef9de4a7cc9fe7ed282 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:02:05 +0200 Subject: [PATCH 28/41] docs(workflow): say a read copy goes back to how the reader found it, and where submodule commits are kept Ignored files stay, the copy is marked when the reader starts so a killed driver still blocks later steps, and submodule commits are kept under refs/teamai/kept-modules//. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 438434800..f0bf8bfa0 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD, which also keeps a submodule's commit; not the stash list, which every worktree shares), then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when an initialized submodule has uncommitted or untracked files, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --ignore-submodules=none` is empty. Otherwise the reader fails `workspace_base`, and a `teamai-unrestored` marker in the worktree's git dir makes `stageLease` refuse later leases there until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//`, since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index df29e0b1e..de5aaf927 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -139,7 +139,7 @@ Use `workflow respond ` for a waiting HITL request, `workfl A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. -Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, the worktree and its submodules are reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. A submodule with uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`, and later steps there fail until you put it back on its branch at that commit with nothing changed and run `teamai workflow retry `. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. +Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, and the submodule commits it made under `refs/teamai/kept-modules//`. The worktree and its submodules are then reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. Ignored files stay in place. A submodule the step changed that has uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`. The workspace is marked when the step starts, so later steps there fail, even after the driver was killed during the step, until you put it back on its branch at that commit as the step found it and run `teamai workflow retry `. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. `workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index f177df515..4d4e67626 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -139,7 +139,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom 没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 -在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败；之后在该工作区的步骤都会失败，直到你把它放回它的分支、停在那个提交且没有任何改动，再运行 `teamai workflow retry `。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 +在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，它在子模块里做的提交保存在 `refs/teamai/kept-modules//` 下。然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。被忽略的文件留在原地。步骤改动过的子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败。步骤开始时工作区就会被标记，所以即使驱动在步骤运行时被杀掉，之后在该工作区的步骤也都会失败，直到你把它放回它的分支、停在那个提交、恢复成步骤开始时的样子，再运行 `teamai workflow retry `。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 `workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 From f01d278d17cfd13d24e360bb225ff60fe856f9f1 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:05:52 +0200 Subject: [PATCH 29/41] refactor(workflow): let putBackCopy own the untouched case and the kept sentence Keeps local-driver.ts under 1,000 lines: clearing the reader's mark when nothing changed and the kept-ref sentence live beside the keep they describe. --- src/workflow/local-driver.ts | 15 ++------------- src/workflow/worktree.ts | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index b0952c516..5b3e9e55a 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -42,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, clearReadCopyMark, commitWork, ensureWorktree, git, headCommit, hooksPath, type KeptChanges, markReadCopy, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, + applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, keptSentence, type KeptChanges, markReadCopy, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -163,7 +163,6 @@ interface Lease { /** Set once the step has a directory to run in. */ cwd?: string; before?: WorktreeSnapshot; - /** A read copy's status before the base overlay, which putting the copy back returns it to. */ found?: string[]; /** Branch agent files saved while the base overlay is in place. */ overlay?: string; @@ -559,18 +558,8 @@ async function leaveReadCopy(lease: Lease): Promise { if (!onBranch && !changed.includes('HEAD')) changed.push('HEAD'); await restoreLeaseOverlay(lease); const read = { branch: workspace.branch, base: workspace.base.commit, commit: before.head }; - if (changed.length === 0) { - await clearReadCopyMark(cwd); - return { read, state: 'untouched' }; - } const put = await putBackCopy({ dir: cwd, branch: read.branch, commit: read.commit, found, changed, attempt: lease.attempt, step: lease.step }); - return put.state === 'restored' ? { read, changed, ...put } : { read, changed, dir: cwd, ...put }; -} - -function keptSentence(kept: KeptChanges): string { - return kept.modules - ? `Its changes are kept on ${kept.ref}, and the submodule commits it made under ${kept.modules}.` - : `Its changes are kept on ${kept.ref}.`; + return put.state === 'unrestored' ? { read, changed, dir: cwd, ...put } : put.state === 'restored' ? { read, changed, ...put } : { read, ...put }; } /** A reader that changed its copy fails. `changed` is the step's own sentence naming the paths. */ diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index cc7cfefe4..3ceb1a189 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -577,7 +577,7 @@ export async function markReadCopy(dir: string, { branch, step }: { branch: stri return { found: status.lines }; } -export async function clearReadCopyMark(dir: string): Promise { +async function clearReadCopyMark(dir: string): Promise { const marker = await gitPath(dir, UNRESTORED_MARKER); if (marker) await fs.rm(marker, { force: true }); } @@ -678,16 +678,26 @@ async function unkeptRepositories(dir: string, { paths, commit }: { paths: strin /** Where a reader's changes were kept: the whole tree on `ref`, and each submodule commit it made under `modules`. */ export type KeptChanges = { ref: string; modules?: string }; +export function keptSentence(kept: KeptChanges): string { + return kept.modules + ? `Its changes are kept on ${kept.ref}, and the submodule commits it made under ${kept.modules}.` + : `Its changes are kept on ${kept.ref}.`; +} + /** - * Puts a copy a reader changed back on `branch` at `commit` with the status `found` it had when the reader started, after + * Puts a copy a reader read back on `branch` at `commit` with the status `found` it had when the reader started, after * keeping what it changed on `refs/teamai/kept/`. A copy it cannot put back keeps its mark with the reason, so * later steps there are refused until it is. */ export async function putBackCopy({ dir, branch, commit, found, changed, attempt, step }: { dir: string; branch: string; commit: string; found: string[]; changed: string[]; attempt: string; step: string; }): Promise< - { state: 'restored'; kept: KeptChanges } | { state: 'unrestored'; reason: string; kept?: KeptChanges } + { state: 'untouched' } | { state: 'restored'; kept: KeptChanges } | { state: 'unrestored'; reason: string; kept?: KeptChanges } > { + if (changed.length === 0) { + await clearReadCopyMark(dir); + return { state: 'untouched' }; + } const unrestored = async (reason: string, kept?: KeptChanges): Promise<{ state: 'unrestored'; reason: string; kept?: KeptChanges }> => { const marker = await gitPath(dir, UNRESTORED_MARKER); const record: UnrestoredMarker = { branch, commit, step, reason, found }; From cef36e5717d4fa7f10588cb8c84a1832b1679c06 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:08:43 +0200 Subject: [PATCH 30/41] test(workflow): a reader that deletes a dirty submodule leaves the copy untouched At 9d4fff61 this was reported as kept while the submodule's uncommitted files were lost. The changed-repositories check already fails it closed. --- src/__tests__/workflow-workspace.test.ts | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 548380c50..b2621d4d9 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -3043,6 +3043,35 @@ steps: expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); }); + it('leaves the copy as the reader left it when it deletes a submodule that held uncommitted files, and says they were NOT saved', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A submodule directory the reader removed cannot be put back without losing what was in it. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); require('fs').writeFileSync('vendor/inside.txt','writer'); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').rmSync('vendor',{recursive:true,force:true})" +`); + addSubmodule(workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const copy = worktreePath(workspace, branchName(runId, 'change', 1)); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ + status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`Its changes were NOT saved and are still in ${copy}`) }, + }); + expect(fs.existsSync(path.join(copy, 'vendor'))).toBe(false); + }); + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build From dfc692068f9dce0da9ab0da4b3fa36a89c302a1d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:15:25 +0200 Subject: [PATCH 31/41] fix(workflow): say a read copy goes back as the step found it, and why a marked copy is refused --- src/workflow/local-driver.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 5b3e9e55a..527cbbc64 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -378,11 +378,11 @@ async function stageLease(lease: Lease, { active, runId }: { active: string; run } const blocked = await unrestoredCopy(worktreePath(active, lease.workspace.branch)); if (blocked?.state === 'unreadable') { - return { code: 'workspace_base', message: `Workspace ${blocked.dir} has a marker teamai cannot read: ${blocked.marker}. Put ${blocked.dir} back on ${lease.workspace.branch} with nothing changed, delete ${blocked.marker}, then retry this step with teamai workflow retry ${runId} ${lease.step}.` }; + return { code: 'workspace_base', message: `Workspace ${blocked.dir} has a marker teamai cannot read: ${blocked.marker}. Put ${blocked.dir} back on ${lease.workspace.branch} as the step found it, delete ${blocked.marker}, then retry this step with teamai workflow retry ${runId} ${lease.step}.` }; } if (blocked) { - const { branch, commit, step } = blocked.marker; - return { code: 'workspace_base', message: `Step "${step}" could not put workspace ${blocked.dir} back, and it is still not back: ${blocked.left}. Put ${blocked.dir} back on ${branch} at ${commit} with nothing changed, then retry this step with teamai workflow retry ${runId} ${lease.step}.` }; + const { branch, commit, step, reason } = blocked.marker; + return { code: 'workspace_base', message: `Step "${step}" read workspace ${blocked.dir} and it was not put back (${reason}). It is still not back: ${blocked.left}. Put ${blocked.dir} back on ${branch} at ${commit} as the step found it, then retry this step with teamai workflow retry ${runId} ${lease.step}.` }; } const opened = await ensureWorktree(active, lease.workspace.branch, lease.workspace.start.commit); if ('error' in opened) return { code: 'workspace_base', message: opened.error }; @@ -572,8 +572,8 @@ function readCopyFailure(left: Exclude, change return { code: 'workspace_base', message: left.kept - ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. ${keptSentence(left.kept)} Later steps there fail until ${left.dir} is back on ${branch} at ${commit} with nothing changed.` - : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit} with nothing changed. Later steps there fail until it is.`, + ? `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. ${keptSentence(left.kept)} Later steps there fail until ${left.dir} is back on ${branch} at ${commit} as the step found it.` + : `${changed} Workspace ${branch} could NOT be put back at ${commit}: ${left.reason}. Its changes were NOT saved and are still in ${left.dir}. Copy them out before you put ${left.dir} back on ${branch} at ${commit} as the step found it. Later steps there fail until it is.`, }; default: { const unhandled: never = left; From b6961d5bcf0ee892c22133433a97c1092ec16aa9 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:31:46 +0200 Subject: [PATCH 32/41] fix(workflow): keep nested submodule commits, and put back only the submodules a reader changed A status line names a path, not its content, so a marker whose copy was dirty when the reader started is no longer cleared by matching status; the person who puts the copy back deletes it. Module pushes ignore push.recurseSubmodules. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 108 +++++++++++++++++++++++ src/workflow/worktree.ts | 49 +++++++--- 5 files changed, 149 insertions(+), 14 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index f0bf8bfa0..a76e34312 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//`, since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive`, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, with `/` written `%2F`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index de5aaf927..c8d1181b1 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -139,7 +139,7 @@ Use `workflow respond ` for a waiting HITL request, `workfl A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. -Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, and the submodule commits it made under `refs/teamai/kept-modules//`. The worktree and its submodules are then reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. Ignored files stay in place. A submodule the step changed that has uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`. The workspace is marked when the step starts, so later steps there fail, even after the driver was killed during the step, until you put it back on its branch at that commit as the step found it and run `teamai workflow retry `. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. +Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, and the submodule commits it made under `refs/teamai/kept-modules//`. The worktree and the submodules the step changed are then reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. Ignored files stay in place. A submodule the step changed that has uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`. The workspace is marked when the step starts, so later steps there fail, even after the driver was killed during the step, until you put it back on its branch at that commit as the step found it and run `teamai workflow retry `. If the workspace already had changes when the step started, also delete the marker file the failure names. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. `workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 4d4e67626..e3c2a9465 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -139,7 +139,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom 没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 -在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，它在子模块里做的提交保存在 `refs/teamai/kept-modules//` 下。然后 worktree 和其中的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。被忽略的文件留在原地。步骤改动过的子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败。步骤开始时工作区就会被标记，所以即使驱动在步骤运行时被杀掉，之后在该工作区的步骤也都会失败，直到你把它放回它的分支、停在那个提交、恢复成步骤开始时的样子，再运行 `teamai workflow retry `。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 +在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，它在子模块里做的提交保存在 `refs/teamai/kept-modules//` 下。然后 worktree 和步骤改动过的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。被忽略的文件留在原地。步骤改动过的子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败。步骤开始时工作区就会被标记，所以即使驱动在步骤运行时被杀掉，之后在该工作区的步骤也都会失败，直到你把它放回它的分支、停在那个提交、恢复成步骤开始时的样子，再运行 `teamai workflow retry `。如果步骤开始时工作区已有改动，还要删除失败信息中给出的标记文件。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 `workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index b2621d4d9..adcb21f85 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -3043,6 +3043,114 @@ steps: expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); }); + it('leaves a nested submodule commit an earlier writer made where it is when a reader changes only a superproject file', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Putting the copy back updates only the submodules the reader changed. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "const cp=require('child_process'),fs=require('fs'); cp.execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init','--recursive']); fs.writeFileSync('vendor/inner/inside.txt','writer'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','writer'],{cwd:'vendor/inner'}); fs.writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').writeFileSync('report.txt','reader')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "const fs=require('fs'); if (fs.readFileSync('vendor/inner/inside.txt','utf8')!=='writer') process.exit(2); fs.writeFileSync('polish.txt','yes')" +`); + const inner = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); + dirs.push(inner); + execFileSync('git', ['init', '-q'], { cwd: inner }); + fs.writeFileSync(path.join(inner, 'inside.txt'), 'one\n'); + execFileSync('git', ['add', 'inside.txt'], { cwd: inner }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: inner }); + addSubmodule(workspace, inner); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty' } }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); + }); + + it('keeps a nested submodule commit the reader recorded in its parent after the copy is removed', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Every submodule commit the reader made is kept, at any depth. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "require('child_process').execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init','--recursive']); require('fs').writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "const cp=require('child_process'),fs=require('fs'); const commit=(cwd)=>cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','reader'],{cwd}); fs.writeFileSync('vendor/inner/inside.txt','reader'); commit('vendor/inner'); commit('vendor')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + const inner = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-sub-')); + dirs.push(inner); + execFileSync('git', ['init', '-q'], { cwd: inner }); + fs.writeFileSync(path.join(inner, 'inside.txt'), 'one\n'); + execFileSync('git', ['add', 'inside.txt'], { cwd: inner }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: inner }); + addSubmodule(workspace, inner); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const test = run.steps.find((step) => step.id === 'change.test'); + expect(test).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty' } }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); + expect(fs.existsSync(worktreePath(workspace, branchName(runId, 'change', 1)))).toBe(false); + const modules = `refs/teamai/kept-modules/${test && 'attempt' in test ? test.attempt : 'none'}`; + expect(git(['show', `${modules}/vendor%2Finner:inside.txt`], workspace)).toBe('reader'); + }); + + it('refuses a later step in a copy whose reader edited a submodule an earlier writer left dirty, though its status line is the same', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: The same status line does not prove a copy that was dirty when the reader started is back. +steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + workspace: continue + command: node -e "const cp=require('child_process'),fs=require('fs'); cp.execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init']); fs.writeFileSync('vendor/inside.txt','writer'); fs.writeFileSync('impl.txt','yes')" + - id: test + needs: [implement] + command: node -e "require('fs').writeFileSync('vendor/inside.txt','reader')" + - id: polish + needs: [implement] + workspace: continue + command: node -e "require('fs').writeFileSync('polish.txt','yes')" +`); + addSubmodule(workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const copy = worktreePath(workspace, branchName(runId, 'change', 1)); + expect(run.steps.find((step) => step.id === 'change.test')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base' } }); + expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('delete') } }); + expect(fs.readFileSync(path.join(copy, 'vendor', 'inside.txt'), 'utf8')).toBe('reader'); + }); + it('leaves the copy as the reader left it when it deletes a submodule that held uncommitted files, and says they were NOT saved', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 3ceb1a189..eba00bae8 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -600,6 +600,10 @@ export async function unrestoredCopy(dir: string): Promise< } const parsed = UnrestoredMarker.safeParse(raw); if (!parsed.success) return { state: 'unreadable', dir, marker: file }; + // A status line names a path, not its content, so only a copy that was clean can be seen to be back. + if (parsed.data.found.length > 0) { + return { state: 'unrestored', dir, marker: parsed.data, left: `it had changes when the step started, so teamai cannot tell when it is back; delete ${file} once it is` }; + } const left = await copyLeftOver(dir, parsed.data); if (left) return { state: 'unrestored', dir, marker: parsed.data, left }; await fs.rm(file, { force: true }); @@ -620,23 +624,41 @@ async function treeGitlinks(dir: string, commit: string): Promise { - const before = await treeGitlinks(dir, from); +/** Submodule paths whose commit `to` records and `from` does not, at any depth below `prefix`. */ +async function movedModules(dir: string, from: string | undefined, to: string, prefix: string): Promise<{ path: string; dir: string; commit: string }[] | { error: string }> { + const before = from ? await treeGitlinks(dir, from) : new Map(); const after = await treeGitlinks(dir, to); if ('error' in before) return before; if ('error' in after) return after; - const moved = [...after].filter(([relative, commit]) => before.get(relative) !== commit); + const moved: { path: string; dir: string; commit: string }[] = []; + for (const [relative, commit] of after) { + const was = before.get(relative); + if (was === commit) continue; + const sub = path.join(dir, relative); + moved.push({ path: `${prefix}${relative}`, dir: sub, commit }); + const nested = await movedModules(sub, was, commit, `${prefix}${relative}/`); + if ('error' in nested) return nested; + moved.push(...nested); + } + return moved; +} + +/** + * Copies each submodule commit `to` records and `from` does not, at any depth, into this repository's own store. A gitlink + * does not keep its commit reachable, and removing the copy deletes the submodule's store with it. + */ +async function keepModuleCommits(dir: string, { from, to, attempt }: { from: string; to: string; attempt: string }): Promise<{ modules?: string } | { error: string }> { + const moved = await movedModules(dir, from, to, ''); + if ('error' in moved) return moved; if (moved.length === 0) return {}; const common = await git(dir, ['rev-parse', '--path-format=absolute', '--git-common-dir']); if (common.code !== 0) return { error: common.stderr.trim() || 'the git directory could not be found' }; const modules = `refs/teamai/kept-modules/${attempt}/`; - for (const [relative, commit] of moved) { - const pushed = await git(path.join(dir, relative), ['push', '-q', common.stdout.trim(), `${commit}:${modules}${relative}`]); - if (pushed.code !== 0) return { error: pushed.stderr.trim() || `the commit in ${relative} could not be kept` }; + for (const module of moved) { + // One ref per path, so vendor and vendor/inner do not clash. Not the user's push policy: this copy stays local. + const ref = `${modules}${module.path.replaceAll('/', '%2F')}`; + const pushed = await git(module.dir, ['push', '-q', '--no-recurse-submodules', common.stdout.trim(), `${module.commit}:${ref}`]); + if (pushed.code !== 0) return { error: pushed.stderr.trim() || `the commit in ${module.path} could not be kept` }; } return { modules }; } @@ -726,7 +748,12 @@ export async function putBackCopy({ dir, branch, commit, found, changed, attempt // A superproject reset leaves initialized submodules where the step moved them. It must not recurse into a submodule an // earlier step left dirty. const noRecurse = ['-c', 'submodule.recurse=false']; - for (const args of [[...noRecurse, 'checkout', '-q', '-f', branch], [...noRecurse, 'reset', '-q', '--hard', commit], ['clean', '-q', '-fd'], ['submodule', '--quiet', 'update', '--checkout', '--recursive']]) { + const gitlinks = await treeGitlinks(dir, commit); + if ('error' in gitlinks) return unrestored(gitlinks.error, kept); + // Only the submodules the step changed: an update of any other would undo what an earlier step left there. + const touched = changed.map((relative) => relative.replace(/\/$/, '')).filter((relative) => gitlinks.has(relative)); + const update = touched.length > 0 ? [['submodule', '--quiet', 'update', '--checkout', '--recursive', '--', ...touched]] : []; + for (const args of [[...noRecurse, 'checkout', '-q', '-f', branch], [...noRecurse, 'reset', '-q', '--hard', commit], ['clean', '-q', '-fd'], ...update]) { const restored = await git(dir, args); if (restored.code !== 0) return unrestored(restored.stderr.trim() || `git ${args.join(' ')} failed`, kept); } From 876d9066ec29a6ebfe56a27b6229fe1446ad6bcb Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 04:36:28 +0200 Subject: [PATCH 33/41] fix(workflow): name kept submodule commits by commit id, and prune whenever a copy's directory is missing --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-workspace.test.ts | 11 +++++++---- src/workflow/worktree.ts | 19 +++++++------------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index a76e34312..33dcd1ebb 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, with `/` written `%2F`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index adcb21f85..4d58b74dc 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2633,7 +2633,8 @@ steps: expect(() => git(['show', `${branch}:nested`], workspace)).toThrow(); const test = run.steps.find((step) => step.id === 'change.test'); const attempt = test && 'attempt' in test ? test.attempt : 'none'; - expect(git(['show', `refs/teamai/kept-modules/${attempt}/nested:a.txt`], workspace)).toBe('x'); + const nested = git(['rev-parse', `refs/teamai/kept/${attempt}:nested`], workspace).trim(); + expect(git(['show', `refs/teamai/kept-modules/${attempt}/${nested}:a.txt`], workspace)).toBe('x'); }); it('fails a reader that created a git repository in its copy unrestored when the repository hides untracked files from status', async () => { @@ -2893,7 +2894,7 @@ steps: expect(reader).not.toBe(recorded); expect(fs.existsSync(worktreePath(workspace, branch))).toBe(false); expect(git(['cat-file', '-t', reader], workspace).trim()).toBe('commit'); - expect(git(['rev-parse', `refs/teamai/kept-modules/${attempt}/vendor`], workspace).trim()).toBe(reader); + expect(git(['rev-parse', `refs/teamai/kept-modules/${attempt}/${reader}`], workspace).trim()).toBe(reader); expect(test).toMatchObject({ failure: { message: expect.stringContaining(`refs/teamai/kept-modules/${attempt}/`) } }); }); @@ -3115,8 +3116,10 @@ steps: expect(test).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty' } }); expect(run.steps.find((step) => step.id === 'change.polish')).toMatchObject({ status: 'succeeded' }); expect(fs.existsSync(worktreePath(workspace, branchName(runId, 'change', 1)))).toBe(false); - const modules = `refs/teamai/kept-modules/${test && 'attempt' in test ? test.attempt : 'none'}`; - expect(git(['show', `${modules}/vendor%2Finner:inside.txt`], workspace)).toBe('reader'); + const attempt = test && 'attempt' in test ? test.attempt : 'none'; + const vendor = git(['rev-parse', `refs/teamai/kept/${attempt}:vendor`], workspace).trim(); + const nested = git(['rev-parse', `refs/teamai/kept-modules/${attempt}/${vendor}:inner`], workspace).trim(); + expect(git(['show', `refs/teamai/kept-modules/${attempt}/${nested}:inside.txt`], workspace)).toBe('reader'); }); it('refuses a later step in a copy whose reader edited a submodule an earlier writer left dirty, though its status line is the same', async () => { diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index eba00bae8..e759af7d4 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -118,15 +118,10 @@ export async function ensureWorktree(main: string, branch: string, start: string const hooks = hooksPath(main); await fs.mkdir(hooks, { recursive: true }); await hideTeamaiWorktrees(main); - const listed = await git(main, ['worktree', 'list', '--porcelain']); - // Git lists the resolved path, which differs from `dir` under a symlinked temp or home directory. - const real = path.join(await fs.realpath(path.dirname(dir)).catch(() => path.dirname(dir)), path.basename(dir)); - const known = listed.code === 0 && listed.stdout.split('\n').some((line) => line.startsWith('worktree ') && [path.resolve(dir), real].includes(path.resolve(line.slice('worktree '.length)))); const present = await fs.stat(dir).then((stats) => stats.isDirectory(), () => false); // A deleted worktree stays registered, and `worktree add` refuses its path until it is pruned. - if (known && !present) await git(main, ['worktree', 'prune']); - const existing = present && (known || await git(dir, ['rev-parse', '--is-inside-work-tree']).then((result) => result.stdout.trim() === 'true').catch(() => false)); - if (existing) { + if (!present) await git(main, ['worktree', 'prune']); + if (present) { const prefix = await git(dir, ['rev-parse', '--show-prefix']); if (prefix.code !== 0 || prefix.stdout.trim() !== '') { return { error: `${dir} is not a workflow worktree, so no step runs there. Move it out of the way, then retry the step.` }; @@ -583,8 +578,8 @@ async function clearReadCopyMark(dir: string): Promise { } /** - * A copy a reader could not put back, while it is still not back. Once it is, the marker is removed, so the person who - * put it back only retries the step. + * A copy a reader could not put back, while it is still not back. Once a copy that was clean is back, the marker is + * removed, so the person who put it back only retries the step. */ export async function unrestoredCopy(dir: string): Promise< { state: 'unrestored'; dir: string; marker: UnrestoredMarker; left: string } | { state: 'unreadable'; dir: string; marker: string } | undefined @@ -655,9 +650,9 @@ async function keepModuleCommits(dir: string, { from, to, attempt }: { from: str if (common.code !== 0) return { error: common.stderr.trim() || 'the git directory could not be found' }; const modules = `refs/teamai/kept-modules/${attempt}/`; for (const module of moved) { - // One ref per path, so vendor and vendor/inner do not clash. Not the user's push policy: this copy stays local. - const ref = `${modules}${module.path.replaceAll('/', '%2F')}`; - const pushed = await git(module.dir, ['push', '-q', '--no-recurse-submodules', common.stdout.trim(), `${module.commit}:${ref}`]); + // Named by commit: a path need not be a valid ref name, and vendor would clash with vendor/inner. This copy stays + // local, so the user's push policy and hooks do not apply. + const pushed = await git(module.dir, ['push', '-q', '--no-verify', '--no-recurse-submodules', common.stdout.trim(), `${module.commit}:${modules}${module.commit}`]); if (pushed.code !== 0) return { error: pushed.stderr.trim() || `the commit in ${module.path} could not be kept` }; } return { modules }; From 4f071ba5df973123febac65314ec04234c10ee67 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 05:34:30 +0200 Subject: [PATCH 34/41] fix(workflow): keep a copy while a step that has not started, or a loop, can still run in it --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-workspace.test.ts | 130 ++++++++++++++++++++++- src/workflow/engine.ts | 21 ++-- src/workflow/workspace.ts | 19 +++- 4 files changed, 159 insertions(+), 13 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 33dcd1ebb..4c15955ca 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, and `plannedBranch` keeps a copy while a reader of it is pending. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no later unsettled leaf still uses that branch; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 4d58b74dc..07c6a9a86 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -16,7 +16,7 @@ import { syncWorkflowDefinitions } from '../workflow/cache.js'; import * as engine from '../workflow/engine.js'; import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; import { runLocalDriver } from '../workflow/local-driver.js'; -import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import { cancelRun, respondToHitl, retryStep } from '../workflow/run-controls.js'; import type { LocalToolAdapter } from '../workflow/local-tool-adapter.js'; import { parseWorkflow } from '../workflow/parse.js'; import { readRun, writeRun } from '../workflow/store.js'; @@ -2382,6 +2382,134 @@ steps: expect(accepted).not.toHaveProperty('data.cleanup'); }); + it.each([ + ['a sub-workflow that needs the writer', `steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + prompt: Implement. + - id: verify + needs: [implement] + workflow: + steps: + - id: test + command: npm test +`, [['change.implement', false], ['change.verify.test', true]]], + ['a sub-workflow that also waits for a step in another copy', `steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + prompt: Implement. + - id: side + needs: [] + workspace: new + command: echo side + - id: verify + needs: [implement, side] + workflow: + steps: + - id: test + command: npm test +`, [['change.implement', false], ['change.side', true], ['change.verify.test', true]]], + ['a fan-out that waits for an answer and reads the writer', `inputs: + items: [string] +steps: + - id: change + workspace: new + workflow: + steps: + - id: fix + workspace: new + prompt: Fix. + - id: gate + needs: [] + approve: { message: Check now? } + - id: check + needs: [fix, gate] + for_each: inputs.items + command: npm test +`, [['change.fix', false], ['change.check[0]', false], ['change.check[1]', true]]], + ['a step loop that runs the writer again', `steps: + - id: change + workspace: new + workflow: + steps: + - id: implement + prompt: Implement. + - id: review + needs: [implement] + command: npm run review + loop: { back_to: implement, while: "length(steps.review.passes) < 2", max_iterations: 2 } +`, [['change.implement', false], ['change.review', false], ['change.implement', false], ['change.review', true]]], + ['a step loop that runs the reporter again', `steps: + - id: change + workspace: new + workflow: + steps: + - id: review + prompt: Review. + loop: { back_to: review, while: "length(steps.review.passes) < 2", max_iterations: 2 } +`, [['change.review', false], ['change.review', true]]], + ['a looping workflow that runs its steps again', `steps: + - id: change + workspace: new + workflow: + loop: { while: "length(steps.implement.passes) < 2", max_iterations: 2 } + steps: + - id: implement + prompt: Implement. +`, [['change.implement', false], ['change.implement', true]]], + ['a loop that opens the copy again on a new branch', `steps: + - id: change + workspace: new + loop: { back_to: change, while: "length(steps.change.passes) < 2", max_iterations: 2 } + workflow: + steps: + - id: implement + prompt: Implement. +`, [['change.implement', true], ['change.implement', true]]], + ] as const)('removes a copy only when no step that can still run uses it: %s', async (_name, steps, expected) => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A copy stays while a step that can still run uses it. +${steps}`, steps.startsWith('inputs:') ? { items: ['a', 'b'] } : undefined); + const owner = driver(workspace); + const base = { commit: git(['rev-parse', 'HEAD'], workspace).trim() }; + const open: Leased[] = []; + const reported: Array<[string, boolean]> = []; + for (let turn = 0; turn < 50; turn += 1) { + const next = await leaseNext(config, runId, owner, { base, workspaceModes: [...modes], hitl: true, running: open.map((lease) => lease.attempt) }); + if ('error' in next) throw new Error(JSON.stringify(next)); + if (next.data.done) break; + if (next.data.lease) { + open.push(next.data.lease as Leased); + continue; + } + const lease = open.shift(); + if (!lease) { + const hitl = next.data.hitl as { step: string } | undefined; + if (!hitl) throw new Error(JSON.stringify(next)); + await respondToHitl(config, runId, { step: hitl.step, choice: 'yes', channel: 'cli', agent: false }); + continue; + } + const contract = lease.workspace; + const to = createHash('sha1').update(`${lease.step}:${turn}`).digest('hex'); + const answer = await reportAttempt(config, runId, { + attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver: owner, + ...(contract?.mode === 'read-only' ? { read: { branch: contract.branch, base: base.commit, commit: base.commit } } : {}), + ...(contract && contract.mode !== 'read-only' ? { changes: { branch: contract.branch, base: base.commit, from: base.commit, to, commits: [to] } } : {}), + }); + if ('error' in answer) throw new Error(JSON.stringify(answer)); + reported.push([lease.step, answer.data.cleanup === true]); + } + + expect(reported).toEqual(expected); + }); + it('runs a test in the copy its implement step changed, and records the commit it read', async () => { const { config, workspace, runId } = await fixture(EXAMPLE); fs.writeFileSync(path.join(workspace, 'verify.js'), "if (require('fs').readFileSync('tracked.txt', 'utf8') !== 'changed\\n') process.exit(1);\n"); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 5528f5ac1..bb12cc594 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1189,13 +1189,18 @@ function workspaceMismatch(report: ReportInput, contract: WorkspaceContract | un return undefined; } -/** A later leaf still runs in `branch`. Workflow and `for_each` steps are not leaves; their children are. */ -function branchStillUsed(run: RunState, branch: string, except: string): boolean { - for (const item of run.steps) { - if (item.id === except || settled(item)) continue; - const located = locateStep(run, item.id); - if (!located || located.step.workflow !== undefined || (located.step.forEach && !isItem(item))) continue; - if (plannedBranch(run, located.scope, item, located.step) === branch) return true; +/** + * Whether a step that can still run uses `branch`. It reads the run as the next `next` settles it, so a loop that runs + * steps again has made them pending, and a workflow or fan-out that can start has added its steps. A running workflow + * or `for_each` step is skipped, since its steps are in the run; one that has not started counts by what it would run. + */ +function branchStillUsed(run: RunState, branch: string): boolean { + const ahead = structuredClone(run); + if ('unreadable' in reconcileRun(ahead)) return true; + for (const item of ahead.steps) { + if (item.status === 'running' || settled(item)) continue; + const located = locateStep(ahead, item.id); + if (located && plannedBranch(ahead, located.scope, item, located.step) === branch) return true; } return false; } @@ -1274,7 +1279,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: : outcome === 'success' ? succeed(step, { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }) : fail(step, { ...settlement, failure }); // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. - const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch, settledStep.id); + const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); return { save: true, envelope: ok({ diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 15c854ba2..eb691f8ef 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -270,8 +270,7 @@ export function planWorkspace( const sessionResult = sessionOf(run, scope, state, definition); if (sessionResult && 'code' in sessionResult) return { mode, fail: sessionResult }; const session = sessionResult ? { session: sessionResult } : {}; - const owner = [...ancestors].reverse().find((ancestor) => ancestor.step.workspace === 'new' - || (ancestor.step.forEach !== undefined && ancestor.step.workspace === 'continue')); + const owner = [...ancestors].reverse().find((ancestor) => opensCopy(ancestor.step)); const modes = request.modes ?? ['read-only']; if (mode === 'read-only') { if (!owner) return { mode, ...session }; @@ -330,8 +329,22 @@ export function planWorkspace( }; } -/** The branch a step would use, when it runs in a workspace. */ +/** Whether the steps inside `step` run in a copy it opens rather than in the one around it. */ +function opensCopy(step: WorkflowStep): boolean { + return step.workspace === 'new' || (step.forEach !== undefined && step.workspace === 'continue'); +} + +/** + * The existing branch a step that has not started yet would run in. A leaf, or a `for_each` step whose items are leaves, + * uses the branch its lease names. The steps of a workflow continue or read the copy around it, unless it opens its own; + * a branch they open themselves does not exist yet. + */ export function plannedBranch(run: RunState, scope: Scope, state: StepState, definition: WorkflowStep): string | undefined { + if (definition.workflow !== undefined) { + if (opensCopy(definition)) return undefined; + const owner = [...ancestorsOf(run, state.id)].reverse().find((ancestor) => opensCopy(ancestor.step)); + return owner && branchName(run.id, owner.instance, owner.pass); + } return planWorkspace(run, scope, state, definition, { driver: { kind: '', machine: '', workspace: '' }, modes: ['read-only', 'new', 'continue'], From 2fe1bb4e3525efe3d7ea82a2e0dbbbd9bcb5bc9f Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 05:39:11 +0200 Subject: [PATCH 35/41] fix(workflow): see submodule changes that ignore settings hide from a step's snapshot, keep, and commit --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 62 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 2 +- src/workflow/worktree.ts | 29 ++++++----- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 4c15955ca..93d8e9df4 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c8d1181b1..bd3ea9855 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -137,7 +137,7 @@ Use `workflow respond ` for a waiting HITL request, `workfl ## Workspaces -A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. +A command with no `workspace` is read-only. Outside a workspace it runs in the current checkout. If it exits 0 and changed repository files, their permissions, a symlink target, the destination of a staged rename, a file inside a dirty submodule, the index, or HEAD, the step fails with `read_only_dirty` and leaves that change in place. A headless prompt with no `workspace` is read-only in the same way, unless an enclosing workflow or `for_each` opens a workspace. An interactive prompt keeps the person's edits, unless that prompt sets `workspace: read-only`. Submodule changes count even when `submodule..ignore` or `diff.ignoreSubmodules` hides them from `git status`: a read-only step that makes one fails, and a step that writes commits the submodule's new commit. Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, and the submodule commits it made under `refs/teamai/kept-modules//`. The worktree and the submodules the step changed are then reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. Ignored files stay in place. A submodule the step changed that has uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`. The workspace is marked when the step starts, so later steps there fail, even after the driver was killed during the step, until you put it back on its branch at that commit as the step found it and run `teamai workflow retry `. If the workspace already had changes when the step started, also delete the marker file the failure names. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index e3c2a9465..fc6d0e032 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -137,7 +137,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom ## 工作区 -没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。 +没有 `workspace` 的命令是只读的。在工作区之外，它在当前检出目录中运行。退出码为 0，且改了仓库文件、文件权限、符号链接目标、已暂存重命名的新路径、已脏子模块里的文件、暂存区或 HEAD 时，步骤以 `read_only_dirty` 失败，改动留在原地。没有 `workspace` 的无头提示词也一样，除非外层工作流或 `for_each` 打开了工作区。交互提示词会保留人的修改，除非该提示词设置了 `workspace: read-only`。即使 `submodule..ignore` 或 `diff.ignoreSubmodules` 让 `git status` 不显示子模块的改动，这些改动也算数：只读步骤做了这类改动会失败，写入的步骤会提交子模块的新提交。 在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，它在子模块里做的提交保存在 `refs/teamai/kept-modules//` 下。然后 worktree 和步骤改动过的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。被忽略的文件留在原地。步骤改动过的子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败。步骤开始时工作区就会被标记，所以即使驱动在步骤运行时被杀掉，之后在该工作区的步骤也都会失败，直到你把它放回它的分支、停在那个提交、恢复成步骤开始时的样子，再运行 `teamai workflow retry `。如果步骤开始时工作区已有改动，还要删除失败信息中给出的标记文件。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 07c6a9a86..a057fc180 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -3311,6 +3311,68 @@ steps: expect(fs.existsSync(path.join(copy, 'vendor'))).toBe(false); }); + it.each([ + ['submodule.vendor.ignore=all in .gitmodules', ['config', '-f', '.gitmodules', 'submodule.vendor.ignore', 'all']], + ['submodule.vendor.ignore=all in the git config', ['config', 'submodule.vendor.ignore', 'all']], + ['diff.ignoreSubmodules=all', ['config', 'diff.ignoreSubmodules', 'all']], + ])('sees what a step changed in a submodule whatever git is told to ignore: %s', async (_name, setting) => { + const init = "cp.execFileSync('git',['-c','protocol.file.allow=always','submodule','update','-q','--init'])"; + const commit = "fs.writeFileSync('vendor/inside.txt','step'); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qam','step'],{cwd:'vendor'})"; + const run = (script: string): string => `node -e "const cp=require('child_process'),fs=require('fs'); ${script}"`; + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A submodule change is seen when git config hides it. +steps: + - id: active + command: ${run("fs.writeFileSync('vendor/inside.txt','step')")} + - id: commits + needs: [] + workspace: new + workflow: + steps: + - id: init + workspace: continue + command: ${run(init)} + - id: read + needs: [init] + command: ${run(commit)} + - id: edits + needs: [] + workspace: new + workflow: + steps: + - id: init + workspace: continue + command: ${run(init)} + - id: read + needs: [init] + command: ${run("fs.writeFileSync('vendor/inside.txt','step')")} + - id: writes + needs: [] + workspace: new + command: ${run(`${init}; ${commit}`)} +`); + const recorded = addSubmodule(workspace); + git(setting, workspace); + git(['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qam', 'ignore', '--allow-empty'], workspace); + + await runLocalDriver(config, runId, workspace); + + const state = await readRun(config, runId); + if (typeof state !== 'object') throw new Error('run missing'); + const step = (id: string) => state.steps.find((item) => item.id === id); + expect(step('active')).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('vendor') } }); + const read = step('commits.read'); + expect(read).toMatchObject({ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('vendor') } }); + const attempt = read && 'attempt' in read ? read.attempt : 'none'; + const kept = git(['rev-parse', `refs/teamai/kept/${attempt}:vendor`], workspace).trim(); + expect(kept).not.toBe(recorded); + expect(git(['rev-parse', `refs/teamai/kept-modules/${attempt}/${kept}`], workspace).trim()).toBe(kept); + expect(step('edits.read')).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining('NOT saved') } }); + expect(step('writes')).toMatchObject({ status: 'succeeded' }); + expect(git(['rev-parse', `${branchName(runId, 'writes', 1)}:vendor`], workspace).trim()).not.toBe(recorded); + }); + it('runs a reader in a copy on the base tool settings, so a hook an earlier step committed there is absent', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 527cbbc64..d6bd2db9c 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -232,7 +232,7 @@ interface WorktreeSnapshot { function gitStatus(workspace: string): Promise<{ ok: boolean; text: string }> { return new Promise((resolve) => { - execFile('git', ['status', '--porcelain', '-z', '--untracked-files=all'], { cwd: workspace, env: process.env }, (error, stdout, stderr) => { + execFile('git', ['status', '--porcelain', '-z', '--untracked-files=all', '--ignore-submodules=none'], { cwd: workspace, env: process.env }, (error, stdout, stderr) => { resolve(error ? { ok: false, text: stderr.trim() } : { ok: true, text: stdout }); }); }); diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index e759af7d4..51b32b6f5 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -347,24 +347,29 @@ async function fileManifest(root: string): Promise> { return files; } +/** Stages every change. `git add -A` skips a submodule whose `ignore` is `all`, so each submodule is also staged by name. */ +async function stageAll(dir: string): Promise { + const added = await git(dir, ['add', '-A']); + if (added.code !== 0) return added; + const listed = await git(dir, ['ls-files', '-s', '-z']); + if (listed.code !== 0) return listed; + const modules = gitlinkLines(listed.stdout).map((line) => line.slice(line.indexOf('\t') + 1)); + return modules.length > 0 ? git(dir, ['add', '-f', '--', ...modules]) : added; +} + export async function commitWork( dir: string, message: string, fromCommit?: string, ): Promise<{ from: string; to: string; commits: string[] } | { error: string }> { const before = fromCommit === undefined ? await git(dir, ['rev-parse', 'HEAD']) : { code: 0, stdout: fromCommit, stderr: '' }; if (before.code !== 0) return { error: before.stderr.trim() || 'HEAD could not be read' }; const from = before.stdout.trim(); - const status = await git(dir, ['status', '--porcelain']); const merging = (await git(dir, ['rev-parse', '--verify', '--quiet', 'MERGE_HEAD'])).code === 0; - if (status.stdout.trim() !== '' || merging) { - if (status.stdout.trim() !== '') { - const added = await git(dir, ['add', '-A']); - if (added.code !== 0) return { error: added.stderr.trim() || 'git add failed' }; - } - const staged = await git(dir, ['diff', '--cached', '--quiet']); - if (staged.code !== 0 || merging) { - const committed = await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit', '-m', message]); - if (committed.code !== 0) return { error: committed.stderr.trim() || committed.stdout.trim() || 'git commit failed' }; - } + const added = await stageAll(dir); + if (added.code !== 0) return { error: added.stderr.trim() || 'git add failed' }; + const staged = await git(dir, ['diff', '--cached', '--quiet', '--ignore-submodules=none']); + if (staged.code !== 0 || merging) { + const committed = await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit', '-m', message]); + if (committed.code !== 0) return { error: committed.stderr.trim() || committed.stdout.trim() || 'git commit failed' }; } const after = await git(dir, ['rev-parse', 'HEAD']); const to = after.stdout.trim(); @@ -727,7 +732,7 @@ export async function putBackCopy({ dir, branch, commit, found, changed, attempt if (dirty.paths.length > 0) return unrestored(`submodules have uncommitted or untracked files: ${dirty.paths.join(', ')}`); // A commit of the whole tree, not `git stash`: the stash list is shared by every worktree, and a stash leaves out a submodule's commit. const ref = `refs/teamai/kept/${attempt}`; - const added = await git(dir, ['add', '-A']); + const added = await stageAll(dir); const tree = added.code === 0 ? await git(dir, ['write-tree']) : added; const committed = tree.code === 0 ? await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit-tree', tree.stdout.trim(), '-p', 'HEAD', '-m', `teamai: kept ${step}`]) From afff388720b947d19e1565ea32754fd93657d7d7 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 05:43:12 +0200 Subject: [PATCH 36/41] fix(workflow): fail a writer that left its branch before committing, reporting, or pushing for it --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-workspace.test.ts | 44 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 5 ++- src/workflow/worktree.ts | 14 +++++--- 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 93d8e9df4..d564503c2 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index bd3ea9855..da3f4492b 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -141,7 +141,7 @@ A command with no `workspace` is read-only. Outside a workspace it runs in the c Inside a workspace an enclosing step opened, a read-only step runs in that workspace, so a test sees the change it follows. It reads the branch its `needs` changed, or the enclosing branch when they changed none. When they changed more than one branch, it fails with `workspace_base`: add a step that merges them, and need that step. It runs one at a time with the workspace's other steps, on the same base agent files, and only a driver that declares `continue` and owns that workspace takes it. It commits nothing. Afterwards the driver puts the workspace back as the step found it: if the step changed it or left its branch, what it changed is kept on `refs/teamai/kept/`, and the submodule commits it made under `refs/teamai/kept-modules//`. The worktree and the submodules the step changed are then reset to the commit it read and its untracked files are removed, and an exit 0 still fails with `read_only_dirty`. Ignored files stay in place. A submodule the step changed that has uncommitted or untracked files cannot be kept, so then nothing is reset. When the workspace is not put back, the step fails with `workspace_base`. The workspace is marked when the step starts, so later steps there fail, even after the driver was killed during the step, until you put it back on its branch at that commit as the step found it and run `teamai workflow retry `. If the workspace already had changes when the step started, also delete the marker file the failure names. The pass records `read`: the branch, the base, and the commit it read. The workspace is a fresh worktree without ignored files such as `node_modules`, so a test installs what it needs, for example `npm ci && npm test`, unless the team's [`postDeploy`](distribution.md#team-scripts) does. -`workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. +`workspace: new` runs the step in its own git worktree, on a branch named for this run, step, and pass. The step commits what it changed. A step that leaves that branch, for example with `git switch`, fails with `workspace_base`: nothing is committed or pushed, and the worktree stays as the step left it. The current checkout is not switched, reset, or cleaned. A later step that lists it in `needs` starts from the first of those branches and merges the rest, in `needs` order. A merge conflict stops there. A command fails with `merge_conflict`, and the message names the branch and the files. A prompt is told which files still conflict and which branches were not merged. `workspace: continue` stays on the branch an enclosing `workflow` step opened with `new`. A prompt with no `workspace` continues that copy. A `for_each` with `workspace: new` gives each item its own branch, so items do not share a worktree. `steps..branches` lists those branches, and each `results` entry has `branch` when the item has one. An item with `after` starts from the branches it waits for. A top-level `workspace: continue`, with no enclosing `new`, fails the step with `workspace_base`. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index fc6d0e032..d40842fab 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -141,7 +141,7 @@ teamai workflow report --json '{"attempt":"","revision":1,"outcom 在外层步骤打开的工作区里，只读步骤在该工作区中运行，因此测试能看到它所跟随的改动。它读取 `needs` 改动过的分支；`needs` 没有改动任何分支时，读取外层分支。`needs` 改动了不止一个分支时，它以 `workspace_base` 失败：添加一个合并这些分支的步骤，并依赖该步骤。它与该工作区的其他步骤一次只运行一个，使用同样的基准 agent 文件，只有声明了 `continue` 且拥有该工作区的驱动才会领取它。它不提交任何内容。结束后，驱动把工作区恢复成步骤开始时的样子：步骤改动了工作区或离开了它的分支时，改动先保存在 `refs/teamai/kept/`，它在子模块里做的提交保存在 `refs/teamai/kept-modules//` 下。然后 worktree 和步骤改动过的子模块重置到它读取的提交，并删除未跟踪文件；退出码为 0 时仍以 `read_only_dirty` 失败。被忽略的文件留在原地。步骤改动过的子模块里有未提交或未跟踪的文件时无法保存，这时什么都不重置。工作区没有恢复时，步骤以 `workspace_base` 失败。步骤开始时工作区就会被标记，所以即使驱动在步骤运行时被杀掉，之后在该工作区的步骤也都会失败，直到你把它放回它的分支、停在那个提交、恢复成步骤开始时的样子，再运行 `teamai workflow retry `。如果步骤开始时工作区已有改动，还要删除失败信息中给出的标记文件。该 pass 记录 `read`：分支、基准和它读取的提交。工作区是新的 worktree，没有 `node_modules` 这类被忽略的文件，所以测试要自己安装依赖，例如 `npm ci && npm test`，除非团队的 [`postDeploy`](distribution.md#团队脚本) 已经安装。 -`workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 +`workspace: new` 在独立的 git worktree 里运行该步骤，分支名带有这次运行、步骤和 pass。步骤会提交它改过的文件。步骤离开了该分支（例如用 `git switch`）时，以 `workspace_base` 失败：不提交也不推送任何内容，worktree 保持步骤离开时的样子。当前检出目录不会被切换、重置或清理。在 `needs` 里列出它的后续步骤，从这些分支的第一个开始，再按 `needs` 的顺序合并其余分支。合并冲突会停在第一处。命令步骤以 `merge_conflict` 失败，消息里写明分支和文件。提示词步骤会被告知哪些文件仍有冲突、哪些分支还没合并。 `workspace: continue` 留在外层 `workflow` 步骤用 `new` 打开的那条分支上。没有 `workspace` 的提示词会沿用这份副本。带 `workspace: new` 的 `for_each` 给每一项自己的分支，因此各项不共用一个 worktree。`steps..branches` 列出这些分支；某一项有分支时，对应的 `results` 条目带 `branch`。带 `after` 的项从它等待的那些分支开始。顶层的 `workspace: continue` 没有外层 `new` 时，步骤以 `workspace_base` 失败。 diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index a057fc180..c469d14e4 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -2887,6 +2887,50 @@ steps: expect(fs.existsSync(path.join(copy, 'polish.txt'))).toBe(false); }); + it.each([ + ['a command that switches to another branch', "cp.execFileSync('git',['switch','-q','-c','scratch']); fs.writeFileSync('edited.txt','yes')", 'on refs/heads/scratch'], + ['a command that detaches HEAD and commits', "cp.execFileSync('git',['checkout','-q','--detach']); fs.writeFileSync('edited.txt','yes'); cp.execFileSync('git',['add','edited.txt']); cp.execFileSync('git',['-c','user.name=Test','-c','user.email=test@example.com','commit','-qm','step'])", 'detached at'], + ['a failing command that switches to another branch', "cp.execFileSync('git',['switch','-q','-c','scratch']); fs.writeFileSync('edited.txt','yes'); process.exit(3)", 'on refs/heads/scratch'], + ['a prompt that switches to another branch', undefined, 'on refs/heads/scratch'], + ])('fails a writer that leaves its branch without committing, reporting, or pushing for that branch: %s', async (_name, script, where) => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A writer's changes reach its branch only from that branch. +steps: + - id: edit + workspace: new + ${script === undefined ? 'prompt: Edit.' : `command: node -e "const cp=require('child_process'),fs=require('fs'); ${script}"`} +`); + const base = git(['rev-parse', 'HEAD'], workspace).trim(); + const origin = path.join(path.dirname(workspace), 'origin.git'); + execFileSync('git', ['init', '-q', '--bare', origin]); + git(['remote', 'add', 'origin', origin], workspace); + config.workflows = { tools: { default: 'claude' }, max_parallel: 1 }; + const adapter: LocalToolAdapter = { + async runInteractive() { + return { started: false, exitCode: 1, failure: { code: 'tool_failed', message: 'interactive is not used' } }; + }, + async run(_tool, input) { + execFileSync('git', ['switch', '-q', '-c', 'scratch'], { cwd: input.cwd }); + fs.writeFileSync(path.join(input.cwd, 'edited.txt'), 'yes'); + return { started: true, exitCode: 0 }; + }, + }; + + await runLocalDriver(config, runId, workspace, { adapter }); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'edit', 1); + const edit = run.steps.find((step) => step.id === 'edit'); + expect(edit).toMatchObject({ status: 'failed', failure: { code: 'workspace_base', message: expect.stringContaining(`${where}`) } }); + expect(edit).toMatchObject({ failure: { message: expect.stringContaining(`not on branch ${branch}`) } }); + expect(edit).not.toHaveProperty('changes'); + expect(git(['rev-parse', branch], workspace).trim()).toBe(base); + expect(git(['log', '--all', '--format=%an'], workspace)).not.toContain('TeamAI'); + expect(git(['for-each-ref', '--format=%(refname)', 'refs/heads'], origin)).toBe(''); + }); + it('creates the copy again when its directory was deleted', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index d6bd2db9c..c15ac4175 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -42,7 +42,7 @@ import type { DriverIdentity, StepHitl, WorkflowEnvelope, WorkflowFailure } from import { isWorkflowError } from './types.js'; import type { WorkspaceContract } from './workspace.js'; import { - applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, keptSentence, type KeptChanges, markReadCopy, mergeBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, + applyBaseConfig, commitWork, ensureWorktree, git, headCommit, hooksPath, keptSentence, type KeptChanges, markReadCopy, mergeBranch, offBranch, pushIfOrigin, putBackCopy, recordConfigOverlay, removeWorktree, reopenConfigConflicts, restoreUneditedConfig, unrestoredCopy, worktreePath, } from './worktree.js'; /** Local lease identity, stable across restarts in the same workspace. */ @@ -586,6 +586,9 @@ async function finishWorkspace(lease: Lease, push: boolean): Promise<{ changes?: if (!lease.workspace || lease.workspace.mode === 'read-only' || !lease.cwd) return {}; if (lease.overlay) await restoreUneditedConfig(lease.cwd, lease.workspace.configCommit, lease.overlay); lease.overlay = undefined; + // A commit on whatever HEAD the step left would be reported and pushed as the leased branch, which would lack it. + const on = await offBranch(lease.cwd, lease.workspace.branch); + if (on) return { failure: { code: 'workspace_base', message: `Step "${lease.step}" left workspace ${lease.cwd} ${on}, not on branch ${lease.workspace.branch}, so nothing was committed or pushed. Check out ${lease.workspace.branch} in ${lease.cwd} with the changes to keep, then retry the step.` } }; const open = await unresolvedConflicts(lease.cwd, lease.conflicts); if (open.length > 0) return { failure: { code: 'merge_conflict', message: `Unresolved merge conflict: ${open.join(', ')}` } }; const committed = await commitWork(lease.cwd, `teamai: ${lease.step}`, lease.rangeFrom); diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 51b32b6f5..f8c2f903f 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -126,11 +126,8 @@ export async function ensureWorktree(main: string, branch: string, start: string if (prefix.code !== 0 || prefix.stdout.trim() !== '') { return { error: `${dir} is not a workflow worktree, so no step runs there. Move it out of the way, then retry the step.` }; } - const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); - if (head !== `refs/heads/${branch}`) { - const on = head ? `on ${head}` : `detached at ${(await git(dir, ['rev-parse', 'HEAD'])).stdout.trim()}`; - return { error: `Workspace ${dir} is ${on}, not on branch ${branch}. Check out ${branch} in ${dir}, then retry the step.` }; - } + const on = await offBranch(dir, branch); + if (on) return { error: `Workspace ${dir} is ${on}, not on branch ${branch}. Check out ${branch} in ${dir}, then retry the step.` }; await worktreeHooks(main, dir, hooks); return { dir }; } @@ -145,6 +142,13 @@ export async function ensureWorktree(main: string, branch: string, start: string return { dir }; } +/** Where HEAD is, such as `on refs/heads/other` or `detached at `, when it is not on `branch`. Undefined when it is. */ +export async function offBranch(dir: string, branch: string): Promise { + const head = (await git(dir, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); + if (head === `refs/heads/${branch}`) return undefined; + return head ? `on ${head}` : `detached at ${(await git(dir, ['rev-parse', 'HEAD'])).stdout.trim()}`; +} + /** Points this worktree's hooks at `hooks` without changing the active checkout's hooks. */ async function worktreeHooks(main: string, dir: string, hooks: string): Promise { await git(main, ['config', 'extensions.worktreeConfig', 'true']); From dedbc1bb229dc106595df9a3e9ea29cf458a99b3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 05:47:47 +0200 Subject: [PATCH 37/41] fix(workflow): lease and start steps in other checkouts while a headless prompt runs --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-local-driver.test.ts | 68 +++++++++++++++++++++ src/workflow/local-driver.ts | 8 ++- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index d564503c2..7e3104357 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -10,7 +10,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **Reconcile the latest report before applying another or handing out a lease.** `reconcileRun`, shared by `leaseNext` and `reportAttempt`, settles every driverless transition to a fixpoint at the run's current revision, through `settleScope` and `settleItems`, including workflow starts, fan-out expansion, render failures and bound exhaustion. `reportAttempt` advances the revision only after it, so only the latest revision can be unreconciled. A reached end applies before a loop decision. Only after reconciliation does `leaseNext` hand back an omitted open lease or issue the first ready leaf in declaration order that this driver can take. A running workflow occupies its parent's position in that order. Settling returns every ready leaf in that order (`ready`), and `leaseNext` walks it. A ready item does not count against `max_parallel` until it is leased, and one `next` leases at most one step, so a held-back item leaves its room to the next one without exceeding the cap. `canTakeLease: false` still reconciles driverless transitions. Every changed run is saved, including a `wait`. - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. -- **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. An interactive prompt uses the terminal and starts only when no command is running. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. +- **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. diff --git a/src/__tests__/workflow-local-driver.test.ts b/src/__tests__/workflow-local-driver.test.ts index eb0aa2911..8de6c5675 100644 --- a/src/__tests__/workflow-local-driver.test.ts +++ b/src/__tests__/workflow-local-driver.test.ts @@ -762,6 +762,74 @@ steps: expect(commandExitedFirst).toEqual([true]); }); + it.each([ + ['a command in another worktree', (mark: (name: string, untilRelease?: boolean) => string) => ` - id: change + workspace: new + prompt: Implement. + - id: lint + needs: [] + workspace: new + command: ${mark('lint')} +`, ['lint'], []], + ['a command in the current checkout', (mark: (name: string, untilRelease?: boolean) => string) => ` - id: change + workspace: new + prompt: Implement. + - id: lint + needs: [] + command: ${mark('lint')} +`, ['lint'], []], + ['a command in the prompt\'s own worktree', (mark: (name: string, untilRelease?: boolean) => string) => ` - id: change + workspace: new + workflow: + steps: + - id: implement + prompt: Implement. + - id: lint + needs: [] + workspace: continue + command: ${mark('lint')} +`, [], ['lint']], + ['a command past max_parallel, counting the prompt', (mark: (name: string, untilRelease?: boolean) => string) => ` - id: change + workspace: new + prompt: Implement. + - id: first + needs: [] + workspace: new + command: ${mark('first', true)} + - id: second + needs: [] + workspace: new + command: ${mark('second')} +`, ['first'], ['second']], + ] as const)('runs a ready step beside a headless prompt only in a free checkout and slot: %s', async (_name, steps, overlaps, waits) => { + const markers = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-markers-')); + dirs.push(markers); + const mark = (name: string, untilRelease = false): string => `node -e "const fs=require('fs'),p=require('path'); fs.writeFileSync(p.join('${markers}','${name}'),'');${ + untilRelease ? ` const t=setInterval(()=>{ if (fs.existsSync(p.join('${markers}','release'))) clearInterval(t); },20);` : ''}"`; + const { config, workspace, runId } = await parallelFixture(`schemaVersion: v1alpha1 +name: build +description: Steps beside a prompt. +steps: +${steps(mark)}`, 2); + const exists = (name: string): boolean => fs.existsSync(path.join(markers, name)); + const seen: string[] = []; + const adapter: LocalToolAdapter = { + runInteractive: vi.fn(), + run: vi.fn(async () => { + for (const deadline = Date.now() + 3000; Date.now() < deadline && !overlaps.every(exists);) await new Promise((resolve) => setTimeout(resolve, 20)); + if (waits.length > 0) await new Promise((resolve) => setTimeout(resolve, 500)); + seen.push(...[...overlaps, ...waits].filter(exists)); + fs.writeFileSync(path.join(markers, 'release'), ''); + return { started: true as const, exitCode: 0 }; + }), + }; + + const done = await runLocalDriver(config, runId, workspace, { adapter, toolPaths }); + + expect(done).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(seen).toEqual(overlaps); + }); + it('drops a held prompt that an end cancels before it starts, and the run goes on', async () => { const { config, workspace, runId } = await parallelFixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index c15ac4175..a38ab6b3d 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -795,7 +795,9 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options const base = await headCommit(cwd); let prompt: { lease: Lease; controller: AbortController; exit: Promise } | undefined; let question: { hitl: LocalHitl; controller: AbortController; exit: Promise<{ kind: 'question'; line?: string }> } | undefined; - const room = (): boolean => !prompt && inFlight.size + dying.size + held.length < maxParallel && !held.some((lease) => lease.prompt !== undefined); + // A headless prompt takes a slot as a command does, so a step in another checkout starts beside it. An interactive one has the terminal. + const busy = (): number => inFlight.size + dying.size + (prompt ? 1 : 0); + const room = (): boolean => !prompt?.lease.interactive && busy() + held.length < maxParallel && !held.some((lease) => lease.prompt !== undefined); const cancel = (attempt: string): void => { if (prompt?.lease.attempt === attempt) prompt.controller.abort(); const index = held.findIndex((lease) => lease.attempt === attempt); @@ -881,7 +883,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options ...[...dying.values()].map(({ lease }) => placeOf(lease, cwd)), ...(prompt ? [placeOf(prompt.lease, cwd)] : []), ]); - for (let index = 0; !prompt?.lease.interactive && held[index]?.command !== undefined && inFlight.size + dying.size < maxParallel;) { + for (let index = 0; !prompt?.lease.interactive && held[index]?.command !== undefined && busy() < maxParallel;) { const lease = held[index]!; if ([...dying.values()].some((stopping) => stopping.step === lease.step) || placesBusy().has(placeOf(lease, cwd)) @@ -898,7 +900,7 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options } inFlight.set(lease.attempt, { lease, command: startCommand(lease.command!, lease.cwd ?? cwd, lease.env ?? {}, detached) }); } - if (!prompt && inFlight.size + dying.size < maxParallel) { + if (!prompt && busy() < maxParallel) { const promptAt = held.findIndex((lease) => lease.prompt !== undefined && !placesBusy().has(placeOf(lease, cwd)) && ![...dying.values()].some((stopping) => stopping.step === lease.step) From cd909273552f6095634f45225d1cd1e56b7067c3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 05:55:04 +0200 Subject: [PATCH 38/41] fix(workflow): fail a read-only step that moves HEAD to another branch or detaches it at the same commit --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-workspace.test.ts | 27 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 9 +++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 7e3104357..10301b488 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index c469d14e4..845aa6fd1 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -195,6 +195,33 @@ steps: expect(fs.readFileSync(path.join(workspace, 'dirty.txt'), 'utf8')).toBe('x'); }); + it.each([ + ['switches to a new branch', "cp.execFileSync('git',['switch','-q','-c','side'])", 'refs/heads/side'], + ['detaches HEAD', "cp.execFileSync('git',['checkout','-q','--detach'])", ''], + ])('fails a command with no workspace when it %s without committing, and leaves HEAD there', async (_name, script, ref) => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: A read-only command that moves HEAD off its branch changes the checkout. +steps: + - id: move + command: node -e "const cp=require('child_process'); ${script}" +`); + + await runLocalDriver(config, runId, workspace); + + expect(await readRun(config, runId)).toMatchObject({ + steps: [{ status: 'failed', failure: { code: 'read_only_dirty', message: expect.stringContaining('HEAD') } }], + }); + const head = ((): string => { + try { + return git(['symbolic-ref', '--quiet', 'HEAD'], workspace).trim(); + } catch { + return ''; + } + })(); + expect(head).toBe(ref); + }); + it('fails a command with no workspace when it commits, and leaves the commit', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index a38ab6b3d..1ac179e13 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -228,6 +228,8 @@ interface WorktreeSnapshot { ok: boolean; files: Map; head?: string; + /** The ref HEAD points to, empty when it is detached, so a switch at the same commit is a change too. */ + branch: string; } function gitStatus(workspace: string): Promise<{ ok: boolean; text: string }> { @@ -283,7 +285,8 @@ async function worktreeMode(workspace: string, relative: string): Promise { const status = await gitStatus(workspace); const head = (await git(workspace, ['rev-parse', 'HEAD'])).stdout.trim(); - if (!status.ok) return { ok: false, files: new Map(), ...(head ? { head } : {}) }; + const branch = (await git(workspace, ['symbolic-ref', '--quiet', 'HEAD'])).stdout.trim(); + if (!status.ok) return { ok: false, files: new Map(), ...(head ? { head } : {}), branch }; const files = new Map(); for (const { mark, relative } of statusEntries(status.text)) { if (files.has(relative)) continue; @@ -295,7 +298,7 @@ async function snapshotWorktree(workspace: string, depth = 0): Promise { @@ -311,7 +314,7 @@ function changedPaths(before: WorktreeSnapshot, after: WorktreeSnapshot): string : !before.ok ? (after.files.size === 0 ? [] : [...after.files.keys()]) : [...new Set([...before.files.keys(), ...after.files.keys()])].filter((name) => before.files.get(name) !== after.files.get(name)); - if (before.head && after.head && before.head !== after.head) files.push('HEAD'); + if ((before.head && after.head && before.head !== after.head) || before.branch !== after.branch) files.push('HEAD'); return files; } From ae8b8606b6aa95db16bbc634dfb046087e8e6741 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 10:09:32 +0200 Subject: [PATCH 39/41] fix(workflow): keep forced branch colors and external diffs out of the driver's git checks With color.branch=always, git branch --contains printed escape codes around the name, so a retry never pushed a branch whose first push failed. With diff.external and diff.trustExitCode, an external diff that called two blobs equal made commitWork skip a writer's staged change. Also tests that status.showUntrackedFiles=no does not hide a writer's new file. --- src/__tests__/workflow-workspace.test.ts | 61 ++++++++++++++++++++++++ src/workflow/worktree.ts | 4 +- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 845aa6fd1..02ff6ec57 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -481,6 +481,49 @@ steps: expect(git(['show', `${branch}:right.txt`], workspace)).toBe('R'); }); + it('commits a writer whose only change is a new file when the repository hides untracked files from status', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: status.showUntrackedFiles=no does not hide a writer's new file. +steps: + - id: change + workspace: new + command: node -e "require('fs').writeFileSync('new.txt','yes')" +`); + git(['config', 'status.showUntrackedFiles', 'no'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const change = run.steps.find((step) => step.id === 'change'); + expect(change && 'changes' in change && change.changes?.commits.length).toBe(1); + expect(git(['show', `${branchName(runId, 'change', 1)}:new.txt`], workspace)).toBe('yes'); + }); + + it('commits a writer whose change an external diff the user trusts calls equal', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: diff.external with diff.trustExitCode does not decide whether there is something to commit. +steps: + - id: change + workspace: new + command: node -e "require('fs').writeFileSync('tracked.txt','clean \\\\n')" +`); + const wrapper = path.join(path.dirname(workspace), 'diff-w.sh'); + fs.writeFileSync(wrapper, '#!/bin/sh\nexec diff -w "$2" "$5"\n', { mode: 0o755 }); + git(['config', 'diff.external', wrapper], workspace); + git(['config', 'diff.trustExitCode', 'true'], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const change = run.steps.find((step) => step.id === 'change'); + expect(change && 'changes' in change && change.changes?.commits.length).toBe(1); + expect(git(['show', `${branchName(runId, 'change', 1)}:tracked.txt`], workspace)).toBe('clean \n'); + }); + it('does not lease a continue step to a driver that does not own the copy', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build @@ -1259,6 +1302,24 @@ steps: expect(await pushIfOrigin(repo, 'teamai/same', [])).toBe('unchanged'); }); + it('pushes a branch only it holds when the user forces branch colors', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-push-')); + dirs.push(root); + const bare = path.join(root, 'origin.git'); + const repo = path.join(root, 'repo'); + execFileSync('git', ['init', '-q', '--bare', bare]); + execFileSync('git', ['init', '-q', '-b', 'main', repo]); + fs.writeFileSync(path.join(repo, 'tracked.txt'), 'one\n'); + execFileSync('git', ['add', 'tracked.txt'], { cwd: repo }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'one'], { cwd: repo }); + execFileSync('git', ['remote', 'add', 'origin', bare], { cwd: repo }); + execFileSync('git', ['checkout', '-q', '-b', 'teamai/solo'], { cwd: repo }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-q', '--allow-empty', '-m', 'two'], { cwd: repo }); + git(['config', 'color.branch', 'always'], repo); + + expect(await pushIfOrigin(repo, 'teamai/solo', [])).toBe('pushed'); + }); + it('commits a hook a step made executable without editing its text', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index f8c2f903f..a109a3d65 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -370,7 +370,7 @@ export async function commitWork( const merging = (await git(dir, ['rev-parse', '--verify', '--quiet', 'MERGE_HEAD'])).code === 0; const added = await stageAll(dir); if (added.code !== 0) return { error: added.stderr.trim() || 'git add failed' }; - const staged = await git(dir, ['diff', '--cached', '--quiet', '--ignore-submodules=none']); + const staged = await git(dir, ['diff', '--cached', '--quiet', '--ignore-submodules=none', '--no-ext-diff']); if (staged.code !== 0 || merging) { const committed = await git(dir, ['-c', 'user.name=TeamAI', '-c', 'user.email=teamai@localhost', 'commit', '-m', message]); if (committed.code !== 0) return { error: committed.stderr.trim() || committed.stdout.trim() || 'git commit failed' }; @@ -419,7 +419,7 @@ async function branchNeedsPush(dir: string, branch: string): Promise { /** True when no other local branch already contains this tip. */ async function tipIsOnlyOn(dir: string, branch: string, tip: string): Promise { - const holders = await git(dir, ['branch', '--contains', tip]); + const holders = await git(dir, ['branch', '--no-color', '--contains', tip]); if (holders.code !== 0) return true; const names = holders.stdout.split('\n').map((line) => line.trim().replace(/^[*+] /, '')).filter((name) => name !== ''); return names.every((name) => name === branch); From ba957ded11ed227d222db490d986d3290bb25666 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 10:19:16 +0200 Subject: [PATCH 40/41] fix(workflow): say where a failed push left a step's commits and how to finish it push_failed carried only git's stderr. It now names the branch and the worktree that hold the commits, and says to push that branch from there or retry the step. Git's own text comes last. --- src/__tests__/workflow-workspace.test.ts | 30 ++++++++++++++++++++++++ src/workflow/local-driver.ts | 3 ++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index 02ff6ec57..f9610d4c9 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -481,6 +481,36 @@ steps: expect(git(['show', `${branch}:right.txt`], workspace)).toBe('R'); }); + it('says where a failed push left the commits, so pushing them from there works', async () => { + const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 +name: build +description: origin refuses the push. +steps: + - id: change + workspace: new + command: node -e "require('fs').writeFileSync('new.txt','yes')" +`); + git(['remote', 'add', 'origin', path.join(path.dirname(workspace), 'missing.git')], workspace); + + await runLocalDriver(config, runId, workspace); + + const run = await readRun(config, runId); + if (typeof run !== 'object') throw new Error('run missing'); + const branch = branchName(runId, 'change', 1); + const copy = worktreePath(workspace, branch); + expect(run.steps.find((step) => step.id === 'change')).toMatchObject({ + status: 'failed', + failure: { + code: 'push_failed', + message: expect.stringMatching(new RegExp(`^Step "change" committed on ${branch} in ${copy}, but pushing it to origin failed\\. The commits stay on ${branch}\\. Fix what blocked the push, then push ${branch} from ${copy} or retry the step\\. Git said: .*missing\\.git`, 's')), + }, + }); + const bare = path.join(path.dirname(workspace), 'missing.git'); + execFileSync('git', ['init', '-q', '--bare', bare]); + git(['push', '-q', 'origin', branch], copy); + expect(git(['show', `${branch}:new.txt`], bare)).toBe('yes'); + }); + it('commits a writer whose only change is a new file when the repository hides untracked files from status', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 1ac179e13..a17226e36 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -605,7 +605,8 @@ async function finishWorkspace(lease: Lease, push: boolean): Promise<{ changes?: to: committed.to, commits: committed.commits, }; - return typeof pushed === 'object' ? { changes, failure: { code: 'push_failed', message: pushed.error } } : { changes }; + if (typeof pushed !== 'object') return { changes }; + return { changes, failure: { code: 'push_failed', message: `Step "${lease.step}" committed on ${lease.workspace.branch} in ${lease.cwd}, but pushing it to origin failed. The commits stay on ${lease.workspace.branch}. Fix what blocked the push, then push ${lease.workspace.branch} from ${lease.cwd} or retry the step. Git said: ${pushed.error}` } }; } async function commandReport(attemptResultPath: string, driver: DriverIdentity, lease: Lease, result: CommandResult): Promise { From 3ef4e1884260306e2e733663d875096458f25503 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Wed, 21 Oct 2026 10:24:16 +0200 Subject: [PATCH 41/41] fix(workflow): do not restore agent files under a parent a step replaced with a symlink The symlinked-parent check ran only before a step. A step that ran mv .github && ln -s .github passed it, and restoreUneditedConfig then followed the new link: it deleted .github/agents files outside the worktree and removed the link. Restoration now skips every agent path whose parent is a symlink, keeping the step's edit. The next step still fails base_config on it. --- src/__tests__/workflow-workspace.test.ts | 26 ++++++++++++++++++++++++ src/workflow/worktree.ts | 21 ++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/__tests__/workflow-workspace.test.ts b/src/__tests__/workflow-workspace.test.ts index f9610d4c9..3ab2934ea 100644 --- a/src/__tests__/workflow-workspace.test.ts +++ b/src/__tests__/workflow-workspace.test.ts @@ -862,6 +862,32 @@ steps: expect(work).toMatchObject({ status: 'failed', failure: { code: 'base_config', message: expect.stringContaining('Commit .github as a directory on the base branch, then start a new run.') } }); }); + it('leaves the target alone when a step replaces .github with a symlink while the base has agent files there', async () => { + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-outside-')); + dirs.push(outside); + const shared = path.join(outside, 'shared'); + const { config, workspace } = await fixture(`schemaVersion: v1alpha1 +name: build +description: Restoring agent files does not follow a parent symlink the step made. +steps: + - id: work + workspace: new + command: node -e "const fs=require('fs'); fs.renameSync('.github',${JSON.stringify(shared).replace(/"/g, "'")}); fs.symlinkSync(${JSON.stringify(shared).replace(/"/g, "'")},'.github')" +`); + fs.mkdirSync(path.join(workspace, '.github', 'agents'), { recursive: true }); + fs.writeFileSync(path.join(workspace, '.github', 'agents', 'a.md'), 'agent'); + execFileSync('git', ['add', '.github'], { cwd: workspace }); + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'agent'], { cwd: workspace }); + const started = await startRun(config, 'build'); + if ('error' in started) throw new Error(started.error.message); + const runId = String(started.data.run); + + await runLocalDriver(config, runId, workspace); + + expect(fs.readFileSync(path.join(shared, 'agents', 'a.md'), 'utf8')).toBe('agent'); + expect(git(['ls-tree', branchName(runId, 'work', 1), '--', '.github'], workspace)).toMatch(/^120000 /); + }); + it('keeps .github workflows from the branch and resets only its agent paths', async () => { const { config, workspace, runId } = await fixture(`schemaVersion: v1alpha1 name: build diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index a109a3d65..011cab919 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -220,11 +220,17 @@ export async function applyBaseConfig( /** An agent path whose parent is a symlink. Copying or removing it would reach outside the worktree. */ async function symlinkedParent(dir: string): Promise<{ relative: string; parent: string } | undefined> { for (const relative of AGENT_PATHS) { - const parts = relative.split('/'); - for (let end = 1; end < parts.length; end += 1) { - const parent = parts.slice(0, end).join('/'); - if (await fs.lstat(path.join(dir, parent)).then((stats) => stats.isSymbolicLink(), () => false)) return { relative, parent }; - } + const parent = await linkedParent(dir, relative); + if (parent) return { relative, parent }; + } + return undefined; +} + +async function linkedParent(dir: string, relative: string): Promise { + const parts = relative.split('/'); + for (let end = 1; end < parts.length; end += 1) { + const parent = parts.slice(0, end).join('/'); + if (await fs.lstat(path.join(dir, parent)).then((stats) => stats.isSymbolicLink(), () => false)) return parent; } return undefined; } @@ -257,8 +263,13 @@ export async function restoreUneditedConfig(dir: string, _commit: string, saved: ...conflictPaths(conflicts).filter((relative) => current[relative] === overlay[relative] && unmerged.has(relative)), ...marked, ])]; + // The step replaced a parent of these agent paths with a symlink. That is its edit, and restoring under it would + // delete or write files outside the worktree. + const linked: string[] = []; + for (const root of AGENT_PATHS) if (await linkedParent(dir, root)) linked.push(path.join(root)); const files = new Set([...Object.keys(overlay), ...Object.keys(current), ...Object.keys(branch)]); for (const relative of files) { + if (linked.some((root) => relative === root || relative.startsWith(`${root}${path.sep}`))) continue; if (unresolved.includes(relative)) continue; if (relative === '_overlay.json' || current[relative] !== overlay[relative]) continue; // The step replaced a parent, for example a directory with a symlink. Do not restore through that edit.