diff --git a/web/lib/flow-onboarding.ts b/web/lib/flow-onboarding.ts index 781d6e40..a46de105 100644 --- a/web/lib/flow-onboarding.ts +++ b/web/lib/flow-onboarding.ts @@ -108,9 +108,9 @@ export function factoryCodeSections(draft: FactoryDraft, target: 'cloud' | 'loca // such a step runs unmetered, so a dollar cap cannot bound it. Wall-clock is // enforced on every step regardless of pricing, which is why it stays the // default here; `{ dollars, wallclock }` together is also valid. Its length - // is the flow's time plan (FLOW_TIME): Cloud's 60-minute cap on a hosted run - // (AgentWorkforce/cloud#4235), and 3h for a local run, which has no sandbox - // lifetime and cannot stop an agent at a limit. + // is the flow's time plan (FLOW_TIME): 2h, an hour under Cloud's 180-minute + // cap on a hosted run (AgentWorkforce/cloud#4270), and 3h for a local run, + // which has no sandbox lifetime and cannot stop an agent at a limit. const budget = `{ wallclock: "${(target === 'cloud' ? FLOW_TIME.headerMinutes : FLOW_TIME.localHeaderMinutes) / 60}h" }`; if (!draft.sources.length) return [{ id: 'empty', code: `import { flow } from "@relayflows/surface"; diff --git a/web/lib/flow-workflows.ts b/web/lib/flow-workflows.ts index 21f414f5..ba71e27f 100644 --- a/web/lib/flow-workflows.ts +++ b/web/lib/flow-workflows.ts @@ -146,23 +146,118 @@ const PERL_LIMITER = * Every limiter stops the check's whole process group and exits 124. The * caller may set `check_dir` (where to run, default here) and `check_out` * (the log). + * + * Some suites need more than one lease: flows#626 (run b71c1687) stopped at + * 840s on a check.sh that rightly mirrors its CI (a kernel release build, + * cargo tests, the schema, the surface gate and the full SDK tests). With + * `RELAYFLOW_CHECK_WAIT` (seconds) set, the check runs detached (setsid, or + * nohup where there is none, every descriptor away from the lease) under the + * in a process group of its own (setsid, else perl's setpgrp), with + * `RELAYFLOW_CHECK_TIMEOUT` as its total, and each call waits at most + * `RELAYFLOW_CHECK_WAIT` for it: `running` means call again. Where neither + * exists nothing could stop the check's descendants, so it is not detached at + * all: it runs in this call under the limiter, within the wait. + * + * With no limiter at all — no `timeout`, no `gtimeout`, no `perl` — nothing + * could bound the suite: it would hold the step's lease until the runner + * killed it, and the run would end with no verdict at all, which is the + * failure this plan exists to prevent. So the check is not started: the + * command reports `unrunnable`, which the report reads as "could not run the + * checks", and names what to install. Cloud's image always has coreutils + * `timeout`, so only a local run can reach this. The waiting call enforces the total: at it, it sends the check's + * whole group SIGTERM, then SIGKILL after a 5s grace, and reports `timeout`. + * The group is the detached shell's own (it records `$$`), so the signal + * reaches the suite and everything it started, not just a wrapper. + * `RELAYFLOW_CHECK_ID` names the attempt. A call with a new ID starts a + * check; one with the ID already started only waits, so a resumed run picks + * up the check it started. One that died without an exit status is a `fail`. + * Without `RELAYFLOW_CHECK_WAIT` the check runs in this call under the + * limiter, as the base-commit check does. + * + * Every finished check leaves its exit status, the seconds it ran and its + * limit next to the log (`$check_out.exit`, `.elapsed`, `.limit`), so the + * report can say how a check failed or how long it ran before it was stopped. + */ +const CHECK_RUNNER = 'cd "$check_dir" || { printf "%s\\n" 126 > "$check_out.status"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; ' + + 'case "$limiter" in (timeout|gtimeout) "$limiter" "$limit" sh "$script" ;; (perl) perl -e "$relayflow_limiter" "$limit" sh "$script" ;; (*) sh "$script" ;; esac > "$check_out" 2>&1 < /dev/null; ' + + 'printf "%s\\n" "$?" > "$check_out.status.tmp"; mv -f "$check_out.status.tmp" "$check_out.status"'; + +/** + * The detached check: it records its own process group, runs the suite in + * it, and keeps a deadline of its own a minute past the total, so a check + * whose flow stopped waiting (an interrupted run) still ends. A check stopped + * for time leaves `.stopped` beside its status, so whoever reads the status, + * a resumed waiter included, reports a timeout. */ +const DETACHED_CHECK_RUNNER = 'printf "%s\\n" "$$" > "$check_out.group"; cd "$check_dir" || { printf "%s\\n" 126 > "$check_out.status"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; ' + + 'sh "$script" > "$check_out" 2>&1 < /dev/null & c=$!; ' + + '( s=$(date +%s); while kill -0 "$c" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > "$check_out.stopped"; printf "%s\\n" 124 > "$check_out.status.tmp"; mv -f "$check_out.status.tmp" "$check_out.status"; kill -KILL "-$$" 2>/dev/null || kill -KILL "$c" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & ' + + 'wait "$c"; st=$?; if [ ! -f "$check_out.stopped" ]; then printf "%s\\n" "$st" > "$check_out.status.tmp"; mv -f "$check_out.status.tmp" "$check_out.status"; fi'; + +/** Starts its arguments in a new process group, deaf to SIGHUP as nohup is. */ +const PERL_GROUP = '$SIG{HUP} = "IGNORE"; setpgrp(0, 0); exec @ARGV or exit 127'; + export const FLOW_CHECK_RUN_COMMAND = [ 'root="$PWD"', `script="\${check_script:-$root/${FLOW_CHECK_SCRIPT}}"`, 'check_dir="${check_dir:-$root}"', 'check_out="${check_out:-$root/.relayflow/check.log}"', 'mkdir -p "$(dirname "$check_out")"', + 'rm -f "$check_out.exit" "$check_out.elapsed" "$check_out.limit"', 'if [ ! -s "$script" ]; then echo "relayflow: there is no check script, so no checks ran." > "$check_out"; echo "relayflow: there is no check script, so no checks ran." >&2; echo none' + '; else limit="${RELAYFLOW_CHECK_TIMEOUT:-840}"; limiter=' + '; if command -v timeout >/dev/null 2>&1; then limiter=timeout; elif command -v gtimeout >/dev/null 2>&1; then limiter=gtimeout; elif command -v perl >/dev/null 2>&1; then limiter=perl; fi' - + `; run_limited() { case "$limiter" in (timeout|gtimeout) "$limiter" "$limit" "$@" ;; (perl) perl -e ${shq(PERL_LIMITER)} "$limit" "$@" ;; (*) "$@" ;; esac; }` - + '; echo "relayflow: running $script in $check_dir" >&2' - + '; ( cd "$check_dir" && unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM && run_limited sh "$script" ) > "$check_out" 2>&1 < /dev/null; status=$?' - + '; tail -n 40 "$check_out" >&2' - + '; if [ "$status" -eq 0 ]; then echo "relayflow: the checks passed." >&2; echo pass' - + '; elif [ -n "$limiter" ] && [ "$status" -eq 124 ]; then echo "relayflow: the checks did not finish within ${limit}s." >&2; echo timeout' - + '; else echo "relayflow: the checks failed with exit $status; the full output is in $check_out." >&2; echo fail; fi' + + `; relayflow_limiter=${shq(PERL_LIMITER)}` + // Only these can put the check in a group of its own, which is what makes + // stopping it, and everything it started, possible. + + '; grouper=; if command -v setsid >/dev/null 2>&1; then grouper=setsid; elif command -v perl >/dev/null 2>&1; then grouper=perl; fi' + + '; relayflow_inline=' + + '; relayflow_check_done() { printf \'%s\\n\' "$1" > "$check_out.exit"; printf \'%s\\n\' "$2" > "$check_out.elapsed"; printf \'%s\\n\' "$limit" > "$check_out.limit"; tail -n 40 "$check_out" >&2' + + '; if [ "$1" -eq 0 ]; then echo "relayflow: the checks passed." >&2; echo pass' + // Exit 124 is a timeout only when this flow stopped the check: the + // limiter's, in this call, or the total's. A suite's own 124 is a failure. + + '; elif [ "$1" -eq 124 ] && { { [ "$relayflow_inline" = yes ] && [ -n "$limiter" ]; } || [ "${3:-}" = stopped ]; }; then echo "relayflow: the checks did not finish within ${limit}s (they ran $2s)." >&2; echo timeout' + + '; else echo "relayflow: the checks failed with exit $1; the full output is in $check_out." >&2; echo fail; fi; }' + // No limiter, so nothing could stop the suite at any limit: it would hold + // this step's lease until the runner killed the step, and the run would + // end with no verdict. The checks are not started at all. + + `; if [ -z "$limiter" ]; then relayflow_unrunnable="relayflow: this machine has no timeout, gtimeout or perl, so there is no way to stop the checks at a time limit and they were not run. Install coreutils (timeout) or perl, then run again."` + + '; printf \'%s\\n\' "$relayflow_unrunnable" > "$check_out"; echo "$relayflow_unrunnable" >&2; echo unrunnable' + + '; elif [ -z "${RELAYFLOW_CHECK_WAIT:-}" ] || [ -z "$grouper" ]; then relayflow_inline=yes' + // Nothing here could stop a detached check's descendants, so the check + // runs in this call instead, within the wait this step may hold its lease, + // where the limiter above stops it. + + '; if [ -n "${RELAYFLOW_CHECK_WAIT:-}" ]; then if [ "$RELAYFLOW_CHECK_WAIT" -lt "$limit" ]; then limit="$RELAYFLOW_CHECK_WAIT"; fi; echo "relayflow: neither setsid nor perl is installed, so the checks cannot be detached; they run in this step under $limiter, limited to ${limit}s." >&2; fi' + + '; echo "relayflow: running $script in $check_dir" >&2; since=$(date +%s)' + + `; ( ${CHECK_RUNNER} ); relayflow_check_done "$(cat "$check_out.status" 2>/dev/null || echo 125)" "$(( $(date +%s) - since ))"; rm -f "$check_out.status"` + + '; else id="${RELAYFLOW_CHECK_ID:-1}"' + // The group leader is the detached shell; until it has said so, the pid + // that started it stands in. + + '; relayflow_group() { cat "$check_out.group" 2>/dev/null || cat "$check_out.pid" 2>/dev/null; }' + + '; relayflow_alive() { g=$(relayflow_group); [ -n "$g" ] && { kill -0 "-$g" 2>/dev/null || kill -0 "$g" 2>/dev/null; }; }' + + '; relayflow_stop() { g=$(relayflow_group); [ -n "$g" ] || return 0; kill -TERM "-$g" 2>/dev/null || kill -TERM "$g" 2>/dev/null; i=0; while relayflow_alive && [ "$i" -lt 5 ]; do sleep 1; i=$((i + 1)); done; kill -KILL "-$g" 2>/dev/null || kill -KILL "$g" 2>/dev/null; return 0; }' + + '; relayflow_status_done() { if [ -f "$check_out.stopped" ]; then relayflow_check_done "$(cat "$check_out.status")" "$1" stopped; else relayflow_check_done "$(cat "$check_out.status")" "$1"; fi; }' + + '; if [ "$(cat "$check_out.id" 2>/dev/null)" != "$id" ]; then' + // A check an earlier attempt left running (an interrupted local run, a + // reused workspace) would write the same files, so it is stopped first. + // `ps` tells the real check from a recycled pid; where there is no `ps`, a + // live group with no status is stopped anyway, because a second check + // writing these files is the worse risk. + + ' if [ ! -f "$check_out.status" ] && relayflow_alive; then' + + ' if ! command -v ps >/dev/null 2>&1; then relayflow_stop; echo "relayflow: stopped what an earlier attempt left running (no ps here to identify it)." >&2' + + '; elif ps -o args= -p "$(relayflow_group)" 2>/dev/null | grep -qF \'check_out.group\'; then relayflow_stop; echo "relayflow: stopped a check an earlier attempt left running." >&2; fi; fi' + + '; rm -f "$check_out.status" "$check_out.status.tmp" "$check_out.stopped" "$check_out.group" "$check_out.pid"; : > "$check_out"; date +%s > "$check_out.since"; printf \'%s\\n\' "$id" > "$check_out.id"' + + '; export check_dir check_out script limit' + + `; if [ "$grouper" = setsid ]; then setsid sh -c ${shq(DETACHED_CHECK_RUNNER)} > /dev/null 2>&1 < /dev/null & else perl -e ${shq(PERL_GROUP)} sh -c ${shq(DETACHED_CHECK_RUNNER)} > /dev/null 2>&1 < /dev/null & fi` + + '; echo "$!" > "$check_out.pid"; echo "relayflow: started $script in $check_dir; it may run ${limit}s in all, and each step waits up to ${RELAYFLOW_CHECK_WAIT}s for it." >&2; fi' + + '; since=$(cat "$check_out.since" 2>/dev/null || date +%s); waited=0' + + '; while [ ! -f "$check_out.status" ] && [ "$waited" -lt "$RELAYFLOW_CHECK_WAIT" ] && relayflow_alive && [ $(( $(date +%s) - since )) -lt "$limit" ]; do sleep 1; waited=$((waited + 1)); done' + + '; elapsed=$(( $(date +%s) - since ))' + + '; if [ -f "$check_out.status" ]; then relayflow_status_done "$elapsed"' + + '; elif relayflow_alive && [ "$elapsed" -lt "$limit" ]; then echo "relayflow: the checks are still running (${elapsed}s of ${limit}s)." >&2; echo running' + + '; elif relayflow_alive; then : > "$check_out.stopped"; printf \'%s\\n\' 124 > "$check_out.status.tmp"; mv -f "$check_out.status.tmp" "$check_out.status"; relayflow_stop; echo "relayflow: stopped the checks after ${elapsed}s, at their ${limit}s." >> "$check_out"; relayflow_check_done 124 "$elapsed" stopped' + + '; else sleep 1; if [ -f "$check_out.status" ]; then relayflow_status_done "$elapsed"; else echo "relayflow: the checks stopped without reporting an exit status." >> "$check_out"; relayflow_check_done 125 "$elapsed"; fi; fi' + + '; fi' + '; fi', ].join('; '); @@ -324,8 +419,10 @@ const CAP_FUNCTION = BLOCK_FUNCTION + '; relayflow_cap() { ' + [ * Writes what the checks found to `.relayflow/check-report.md`, and the pull * request body — summary.md followed by that report — to `.relayflow/pr-body.md`. * - * The caller sets `check` (the branch's token, or `skipped` when there was no - * time to run the checks), `baseline` (the base commit's token, empty when it + * The caller sets `check` (the branch's token, `skipped` when there was no + * time to run the checks, or `unrunnable` when there was no way to bound + * them: failed, timed out, not run and could-not-run each say so, a failure + * with its exit status, a timeout with how long it ran against its budget), `baseline` (the base commit's token, empty when it * was not needed, or `skipped` when there was no time to check it), and * `implementer_timeout=yes` when the implementer was stopped at its time * limit, and `reference` to the closing reference line the next step adds. The base commit's output is shown whenever it ran, including when @@ -351,22 +448,32 @@ export const FLOW_CHECK_REPORT_COMMAND = [ `if [ -s .relayflow/check.log ]; then tail -n 80 .relayflow/check.log > "$parts/check"; relayflow_cap "$parts/check" ${REPORT_SECTION_LIMITS.log} tail ".relayflow/check.log in the run workspace"; fi`, `if [ -s .relayflow/base-check.log ]; then tail -n 80 .relayflow/base-check.log > "$parts/base"; relayflow_cap "$parts/base" ${REPORT_SECTION_LIMITS.log} tail ".relayflow/base-check.log in the run workspace"; fi`, `if [ -s .relayflow/repair-notes.md ]; then cp .relayflow/repair-notes.md "$parts/repair"; relayflow_cap "$parts/repair" ${REPORT_SECTION_LIMITS.repair} head ".relayflow/repair-notes.md in the run workspace"; fi`, + // How the last branch check ended (FLOW_CHECK_RUN_COMMAND): its exit status, + // and how long it ran against its limit, in whole seconds. + 'relayflow_number() { v=$(cat "$1" 2>/dev/null | head -n 1); case "$v" in (""|*[!0-9]*) ;; (*) printf %s "$v" ;; esac; }', + 'relayflow_duration() { if [ "$1" -ge 60 ]; then printf "%dm%02ds" $(($1 / 60)) $(($1 % 60)); else printf "%ds" "$1"; fi; }', + 'ck_exit=$(relayflow_number .relayflow/check.log.exit); ck_elapsed=$(relayflow_number .relayflow/check.log.elapsed); ck_limit=$(relayflow_number .relayflow/check.log.limit)', '{ printf \'## Checks\\n\\n\'' + `; if [ "\${implementer_timeout:-}" = yes ]; then printf '%s\\n\\n' "**The implementer was stopped at its time limit**, so this change may be incomplete. Its committed work is what was checked and pushed; the pull request is a draft."; fi` + '; case "$check" in' + ` pass) printf '%s\\n' "Relayflow ran this repository's checks (${FLOW_CHECK_SCRIPT}) and they passed." ;;` + ` skipped) printf '%s\\n' "**Relayflow ran out of time before it could run this repository's checks** (${FLOW_CHECK_SCRIPT}), so no checks ran on this change. The pull request is a draft; run the checks before merging." ;;` + + ` unrunnable) printf '%s\\n' "**Relayflow could not run this repository's checks** (${FLOW_CHECK_SCRIPT}): the machine it ran on has no \\\`timeout\\\`, \\\`gtimeout\\\` or \\\`perl\\\`, so there was no way to stop the suite at a time limit, and starting it could have ended the run with no verdict at all. This is not a test failure, and nothing here says the change is broken. Install coreutils (for \\\`timeout\\\`) or perl and run the flow again, or run ${FLOW_CHECK_SCRIPT} by hand. The pull request is a draft; run the checks before merging." ;;` + ` none) printf '%s\\n' "Relayflow found no way to run this repository's tests — no check command, no CI test job it could follow, no test target and no recognised project file — so no checks ran. Review the change with that in mind." ;;` - + ' *) case "$baseline" in' - + ` fail|timeout) printf '%s\\n' "**The checks fail on the base commit too**, so these failures were not introduced by this change: they come from the repository itself or from the environment the checks ran in. This pull request is a draft until someone looks." ;;` + + ' *) if [ "$check" = timeout ]; then' + + ` if [ -n "$ck_elapsed" ] && [ -n "$ck_limit" ]; then printf '%s\\n\\n' "**The checks timed out**: they ran for $(relayflow_duration "$ck_elapsed") of their $(relayflow_duration "$ck_limit") budget and were stopped before they finished, so this is a time limit, not a test failure. Run them locally (\\\`sh ${FLOW_CHECK_SCRIPT}\\\`) to see how long they take, or raise the check budget (\\\`checkTotal\\\` in the flow) and run the flow again."` + + `; else printf '%s\\n\\n' "**The checks timed out**: they were stopped at their time budget before they finished, so this is a time limit, not a test failure. Run them locally (\\\`sh ${FLOW_CHECK_SCRIPT}\\\`) to see how long they take, or raise the check budget (\\\`checkTotal\\\` in the flow) and run the flow again."; fi` + + `; else if [ -n "$ck_exit" ]; then printf '%s\\n\\n' "**The checks failed** (exit $ck_exit). The end of their output is below."; else printf '%s\\n\\n' "**The checks failed.** The end of their output is below."; fi; fi` + + '; case "$baseline" in' + + ` fail|timeout) printf '%s\\n' "**The checks fail on the base commit too**, so this was not introduced by this change: it comes from the repository itself or from the environment the checks ran in. This pull request is a draft until someone looks." ;;` + ` pass) printf '%s\\n' "**This change breaks checks that pass on the base commit.** The flow tried to repair it and could not. The pull request is a draft so the work is not lost; it is not ready to merge." ;;` + ` new) printf '%s\\n' "**The checks this change adds fail.** The repository had no checks before this change, so there is no base commit to compare with; the flow tried to repair them and could not. The pull request is a draft so the work is not lost; it is not ready to merge." ;;` - + ` skipped) printf '%s\\n' "**The checks failed**, and there was no time left in the run to check the base commit (base not checked), so it is not known whether this change caused them. This pull request is a draft until someone looks." ;;` - + ` *) printf '%s\\n' "**The checks failed**, and the base commit could not be checked for comparison, so it is not known whether this change caused them. This pull request is a draft until someone looks." ;;` + + ` skipped) printf '%s\\n' "There was no time left in the run to check the base commit (base not checked), so it is not known whether this change caused this. This pull request is a draft until someone looks." ;;` + + ` *) printf '%s\\n' "The base commit could not be checked for comparison, so it is not known whether this change caused this. This pull request is a draft until someone looks." ;;` + ' esac ;;' + ' esac' + `; if [ -s "$parts/script" ]; then printf '\\n
What ran (${FLOW_CHECK_SCRIPT})\\n\\n'; relayflow_block "$parts/script" sh; printf '
\\n'; fi` - + `; if [ "$check" != pass ] && [ "$check" != none ] && [ "$check" != skipped ] && [ -s "$parts/check" ]; then printf '\\n
Output on this branch (last 80 lines)\\n\\n'; relayflow_block "$parts/check" ''; printf '
\\n'; fi` + + `; if [ "$check" != pass ] && [ "$check" != none ] && [ "$check" != skipped ] && [ "$check" != unrunnable ] && [ -s "$parts/check" ]; then printf '\\n
Output on this branch (last 80 lines)\\n\\n'; relayflow_block "$parts/check" ''; printf '
\\n'; fi` + `; if [ "$baseline" = fail ] || [ "$baseline" = timeout ] || [ "$baseline" = unknown ]; then if [ -s "$parts/base" ]; then printf '\\n
Output on the base commit (last 80 lines)\\n\\n'; relayflow_block "$parts/base" ''; printf '
\\n'; fi; fi` + `; if [ -s "$parts/repair" ]; then printf '\\n### What the repair agent found\\n\\n'; cat "$parts/repair"; fi` + '; } > "$report"', @@ -413,12 +520,17 @@ export function flowCommentChangeCommand(file: string): string { * it only what is left once the steps behind it fit: every long step has a * limit, and every limit leaves room to publish. * - * Cloud caps a hosted run at 60 minutes, the verified lifetime of an E2B - * sandbox (AgentWorkforce/cloud#4235), and that budget also pays for the - * sandbox setup. Measured Garden runs took 1.5-2h under the old 3h plan - * (agentrelay.com#155), so the plan is cut to fit: no separate planner or plan - * reviewer (the implementer plans), one repair round, one review and no fix - * round. A typical run takes 30-45 minutes and the worst case fits in 60. + * Cloud capped a hosted run at 60 minutes (AgentWorkforce/cloud#4235), and + * measured Garden runs took 1.5-2h under the old 3h plan (agentrelay.com#155), + * so the plan was cut to fit: no separate planner or plan reviewer (the + * implementer plans), one review and no fix round. Cloud now honours a header + * of up to 180 minutes (AgentWorkforce/cloud#4270), and the plan stays the + * same size: the extra time is margin, not a longer plan. The implementer is + * done by the minute the 60-minute plan ended it (buildByMinutes), and what + * follows goes to recovery: a second repair round only when the checks still + * fail, longer limits on the steps seen to hit theirs, and room for a slow run + * to publish and report instead of stopping for time. A typical run still + * takes 30-45 minutes; only a run that goes wrong uses more. * * Since relayflows 2.0.40 (AgentWorkforce/flows#606) an agent step takes a * hard `timeout`, at most agentLimitMaxMinutes; the cloud flow states one for @@ -427,32 +539,52 @@ export function flowCommentChangeCommand(file: string): string { * sandbox lifetime, and an unbounded agent needs the slack. See workflowCode. */ export const FLOW_TIME = (() => { - const headerMinutes = 60; // Cloud's cap on a hosted run (AgentWorkforce/cloud#4235). + const headerMinutes = 120; // An hour under Cloud's 180-minute cap on a hosted run (AgentWorkforce/cloud#4270). const localHeaderMinutes = 180; // Cloud's run budget also counts the setup before the body: an E2B cold // start (about 3 minutes for the full template), the clone, and the agent // CLIs' first start. const setupMinutes = 8; - // `f.run` leases a command for at most 15 minutes; the check stops itself - // at 14 (FLOW_CHECK_RUN_COMMAND), or sooner when less of the run is left. + // `f.run` leases a command for at most 15 minutes; the base-commit check + // stops itself within one (FLOW_CHECK_RUN_COMMAND), and each wait for a + // branch check takes at most checkLimitMinutes of one. const checkMinutes = 15; const checkLimitMinutes = 14; + // A branch check runs detached and is waited on lease after lease, up to + // this in all: b71c1687's CI-shaped check.sh ran past 14 (flows#626). + const checkTotalMinutes = 30; + // The most waits one branch check takes; each may hold its lease a minute + // past its wait. + const checkPolls = Math.ceil((checkTotalMinutes + 1) / checkLimitMinutes); // Below these, a step is not worth starting: the run skips it and says so. const checkFloorMinutes = 5; const repairFloorMinutes = 5; const reviewFloorMinutes = 8; - const repairMinutes = 15; + // Repairs took 19m27s and 37m18s in bda21b91; a 15m limit would have + // stopped the first just short of done. Cloud only: a local run keeps 15. + const repairMinutes = 20; + const localRepairMinutes = 15; + // A second round runs only when the re-check still fails and the first + // repair finished, and only when it leaves the review and the base check + // their time. Cloud only: a local agent has no limit to stop it. + const repairRounds = 2; + const localRepairRounds = 1; const reviewMinutes = 20; // Check discovery reads CI configuration and writes one script; it took - // 6m and 9m24s in bda21b91. A discovery stopped at its limit falls back to - // the ecosystem default, so a hard stop costs little. - const discoveryMinutes = 10; + // 6m and 9m24s in bda21b91, against a 10m limit. A discovery stopped at its + // limit falls back to the ecosystem default, so a hard stop costs little. + // It runs inside buildByMinutes, so this never lengthens the plan. + const discoveryMinutes = 15; // The prototype workflow's parallel agents are each charged in full, so // three of them cost three times their wall clock. const prototypeMinutes = 10; const comparatorMinutes = 5; // What the steps before the implementer always leave it. const implementerFloorMinutes = 10; + // The minute of the body by which discovery, the prototypes and the + // implementer are done: where the 60-minute plan ended them (its 52m body + // less its 18m publishing reserve), so the build is no longer than before. + const buildByMinutes = 34; // relayflows refuses an agent timeout above 60m (AgentWorkforce/flows#606). const agentLimitMaxMinutes = 60; // A step with no timeout of its own gets the kernel's 30s default, and a @@ -463,29 +595,34 @@ export const FLOW_TIME = (() => { // forgeMinutes; the draft-and-comment follow-ups get followUpMinutes; a push // gets pushMinutes, because when GitHub refuses workflow edits // FLOW_PUSH_COMMAND rebuilds the commits, pushes a second time and appends - // the withheld patch, all in the same step. - const forgeMinutes = 3; + // the withheld patch, all in the same step. A push to a large repository + // outran its limit in ccbc27c8; 5m (was 3m) leaves a slow one room. + const forgeMinutes = 5; const followUpMinutes = 1; const pushMinutes = 2 * forgeMinutes + followUpMinutes; const defaultStepMinutes = 0.5; // What can follow the review: one draft-and-comment follow-up (time stop // or review blocked: 1m) and the review-findings report (0.5m). const closeMinutes = followUpMinutes + defaultStepMinutes; - // Publishing at the longest each step may take: dropping working files (3m), - // the push (7m) and opening the change request (3m), the four local steps + // Publishing at the longest each step may take: dropping working files (5m), + // the push (11m) and opening the change request (5m), the four local steps // between them at the default (publish check, check report, metadata, - // validation: 4 x 0.5m), then closing (1.5m). 16.5m, rounded up to 17m. + // validation: 4 x 0.5m), then closing (1.5m). 24.5m, rounded up to 25m. const publishMinutes = Math.ceil(forgeMinutes + pushMinutes + forgeMinutes + 4 * defaultStepMinutes + closeMinutes); return Object.freeze({ headerMinutes, localHeaderMinutes, setupMinutes, - checkMinutes, checkLimitMinutes, checkFloorMinutes, repairMinutes, repairFloorMinutes, reviewMinutes, reviewFloorMinutes, - discoveryMinutes, prototypeMinutes, comparatorMinutes, implementerFloorMinutes, agentLimitMaxMinutes, + checkMinutes, checkLimitMinutes, checkTotalMinutes, checkPolls, checkFloorMinutes, repairMinutes, localRepairMinutes, repairRounds, localRepairRounds, repairFloorMinutes, reviewMinutes, reviewFloorMinutes, + discoveryMinutes, prototypeMinutes, comparatorMinutes, implementerFloorMinutes, buildByMinutes, agentLimitMaxMinutes, forgeMinutes, pushMinutes, followUpMinutes, defaultStepMinutes, closeMinutes, publishMinutes, bodyMinutes: headerMinutes - setupMinutes, localBodyMinutes: localHeaderMinutes - setupMinutes, // What a long step before the pull request leaves behind it: publishing, // and the clock read and check resolution between them. publishReserveMinutes: publishMinutes + 2 * defaultStepMinutes, + // What the steps up to the implementer leave behind them: everything after + // buildByMinutes, which always holds a check at its whole budget, + // publishing and the review's floor. + buildReserveMinutes: headerMinutes - setupMinutes - buildByMinutes, // What the review leaves behind it once the pull request is open: clearing // the last review's files, reading review.clean, and closing. reviewReserveMinutes: 2 * defaultStepMinutes + closeMinutes, @@ -888,7 +1025,7 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType Math.min(cap, Math.floor(await minutesLeft() - after)); // Publishing, and the steps between a long step and it. const publishing = ${t.publishReserveMinutes}; + // What the steps up to the implementer leave: on Cloud, everything after + // minute ${t.buildByMinutes}, so the build takes no longer than it did in 60 minutes. + const building = ${cloud ? t.buildReserveMinutes : t.publishReserveMinutes}; // A long agent step may be stopped at its time limit. Its step then resolves // with completionReason "timeout" instead of throwing, and what the agent // committed stays. That is never success: each caller says what it means. @@ -1000,8 +1140,8 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType f.agent("prototype-" + (index + 1), { cli: prototypeAgents[index], @@ -1012,7 +1152,7 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType ["pass", "fail", "timeout", "none"].includes(value.trim()) ? value.trim() : "fail"; + // report. Each branch check gets up to checkTotal minutes, what is left once + // publishing and the review's floor fit, and stops there. + const verdict = (value: string) => ["pass", "fail", "timeout", "none", "unrunnable"].includes(value.trim()) ? value.trim() : "fail"; const verdictOf = (value: string) => /^[a-z]*$/.test(value) ? value : "unknown"; const broken = (value: string) => value === "fail" || value === "timeout"; const runChecks = ${JSON.stringify(FLOW_CHECK_RUN_COMMAND)}; @@ -1064,35 +1205,73 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType { + const from = await clock(); + const id = startedAt + "-" + (++checkRuns); + let result = "running"; + for (let waited = 0; result === "running" && waited <= minutes; ) { + const wait = Math.min(${t.checkLimitMinutes}, minutes + 1 - waited); + waited += wait; + result = (await f.run("RELAYFLOW_CHECK_ID=" + id + "; RELAYFLOW_CHECK_TIMEOUT=" + minutes * 60 + "; RELAYFLOW_CHECK_WAIT=" + wait * 60 + "; " + runChecks, { timeout: wait + 1 + "m" })).trim(); + } + checkTook = (await clock() - from) / 60; + return result === "running" ? "timeout" : result; + }; + // Behind each check: the two clock reads around it and each wait's minute + // of lease beyond it, then publishing, and the review's floor where there + // is one (its whole allowance locally, where nothing stops it sooner). + const checkSlack = ${1 + t.checkPolls}; + const reviewing = ${workflow === 'simple' ? 0 : `${floor(t.reviewFloorMinutes, t.reviewMinutes) + t.reviewReserveMinutes}`}; let check = "skipped"; - const checkLimit = await allowance(${t.checkLimitMinutes}, 2 + publishing); + const checkLimit = await allowance(checkTotal, checkSlack + publishing + reviewing); if (checkLimit < ${t.checkFloorMinutes}) { console.error("Skipped the checks: too little of the flow's time budget is left to run them and still publish. The pull request opens as a draft that says no checks ran."); } else { - check = verdict(await timedCheck("", runChecks, checkLimit)); + check = verdict(await spannedCheck(checkLimit)); + // No way to bound the suite, so it was never started: not a verdict on + // this change, and nothing to repair or compare. The draft says so. + if (check === "unrunnable") console.error("The checks could not be run: this machine has no timeout, gtimeout or perl to stop them at a time limit, so starting them could have ended the run with no verdict. The pull request opens as a draft that says so."); } - if (broken(check)) { - // One repair round: the repair, freeing disk, the re-check and publishing - // must fit in what is left. - const recheck = Math.min(${t.checkLimitMinutes}, Math.ceil(checkTook) + 1); - const repairLimit = await allowance(${t.repairMinutes}, recheck + 3 + publishing); - if (repairLimit < ${floor(t.repairFloorMinutes, t.repairMinutes)}) { - console.error("Skipped the repair: too little of the flow's time budget is left to repair and still publish. The checks stand as they are."); - } else { - const repair = await f.agent("check-repair", { - ${options('check-repair', 'builder', '', 'repairLimit')} - }); - await f.run(freeDisk); - const recheckLimit = await allowance(${t.checkLimitMinutes}, 2 + publishing); - if (recheckLimit >= 1) check = verdict(await timedCheck("", runChecks, recheckLimit)); - else console.error("No time was left to check the repair again; the checks stand as they were before it."); - if (timedOut(repair)) { - // A failed attempt: what it committed was checked again above, and - // whatever still fails takes the draft-and-report path below. - console.error("check-repair was stopped at its " + repairLimit + "m limit, so it counts as a failed repair attempt. Its committed work is kept and was checked again (" + check + ")."); - } + // A second repair round runs only for checks that still fail after the + // first, and leaves the review, where there is one, its floor: it is + // recovery, never a longer plan. + // The base-commit check that may follow, as compareWithBase budgets it: + // nothing when it would be skipped anyway. + const baseReserve = () => checkPlan === "none" || Math.ceil(checkTook) > ${t.checkMinutes - 2} ? 0 : Math.max(${t.checkFloorMinutes}, Math.ceil(checkTook)) + 5; + for (let round = 1; round <= ${cloud ? t.repairRounds : t.localRepairRounds} && broken(check); round++) { + // Each round: the repair, freeing disk, the re-check, publishing and the + // review must fit in what is left; a second round also keeps the base + // check's time, and its re-check is held to what was kept for it. + const recheck = Math.min(checkTotal, Math.ceil(checkTook) + 1); + const baseHold = round > 1 ? baseReserve() : 0; + const repairLimit = await allowance(${cloud ? t.repairMinutes : t.localRepairMinutes}, recheck + 1 + checkSlack + publishing + reviewing + baseHold); + if (repairLimit < ${floor(t.repairFloorMinutes, cloud ? t.repairMinutes : t.localRepairMinutes)}) { + console.error(round === 1 + ? "Skipped the repair: too little of the flow's time budget is left to repair and still publish. The checks stand as they are." + : "Skipped the second repair: too little of the flow's time budget is left to repair again and still publish and review. The checks stand as they are."); + break; + } + const repair = await f.agent("check-repair" + (round === 1 ? "" : "-" + round), { + ${options('check-repair', 'builder', '', 'repairLimit')} + }); + await f.run(freeDisk); + const recheckLimit = await allowance(checkTotal, checkSlack + publishing + reviewing + baseHold); + if (recheckLimit >= 1) check = verdict(await spannedCheck(recheckLimit)); + else console.error("No time was left to check the repair again; the checks stand as they were before it."); + if (timedOut(repair)) { + // A failed attempt: what it committed was checked again above, and + // another repair of the same failures would most likely run out too, + // so whatever still fails takes the draft-and-report path below. + console.error("check-repair was stopped at its " + repairLimit + "m limit, so it counts as a failed repair attempt. Its committed work is kept and was checked again (" + check + "); no further repair is tried."); + break; } } // Only a failure pays for the comparison: pass, fail, timeout or unknown. @@ -1108,7 +1287,7 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType { it('pins every Claude step in the simple prebuilt flow to a probeable model', async () => { const calls = await execute({ ...draft, agents: ['claude'], workflow: 'simple' }); - expect(Object.keys(calls)).toEqual(['check-discovery', 'implementer', 'check-repair']); + // Checks that still fail after the first repair get a second round. + expect(Object.keys(calls)).toEqual(['check-discovery', 'implementer', 'check-repair', 'check-repair-2']); for (const options of Object.values(calls)) { expect(options).toMatchObject({ cli: 'claude', model: 'claude-sonnet-5' }); } diff --git a/web/lib/test/flow-budget.test.ts b/web/lib/test/flow-budget.test.ts index c19e31f7..07589eb9 100644 --- a/web/lib/test/flow-budget.test.ts +++ b/web/lib/test/flow-budget.test.ts @@ -19,7 +19,25 @@ import { factorySource, type FactoryDraft } from '../flow-onboarding'; // checks passing at step 14), because one check run takes 11-14 min on that // repository and two repair passes took 57 min. AgentWorkforce/cloud#4235 then // capped every hosted run at 60 minutes, the lifetime of an E2B sandbox, while -// measured Garden runs took 1.5-2h (agentrelay.com#155). +// measured Garden runs took 1.5-2h (agentrelay.com#155). AgentWorkforce/cloud#4270 +// raised the cap to 180 minutes; the plan keeps its 60-minute length and the +// rest is recovery room. + +/** Cloud's most for a hosted run's declared wallclock (AgentWorkforce/cloud#4270). */ +const CLOUD_CAP_MINUTES = 180; + +/** + * The happy path's planned allowances under the 60-minute plan (ceb3900): the + * build done by minute 34 of its 52m body, one 14m check, 17m of publishing + * and a 20m review. 85m. + */ +const SIXTY_MINUTE_PLAN_MINUTES = 34 + 14 + 17 + 20; +/** + * What the plan may plan now: 109m. Only waits on work outside the agents + * grew: one check may run 30m (flows#626) and publishing 25m (slow pushes). + * The agents' own allowances, the build window and the review, did not. + */ +const PLANNED_CEILING_MINUTES = 34 + 30 + 25 + 20; const issue = { source: 'github', title: 'Fix login', body: 'Login fails', labels: ['ready'], repository: 'acme/app', identifier: '#507', url: 'https://github.com/acme/app/issues/507' }; const draft: FactoryDraft = { version: 4, sources: ['github'], sourceSettings: { github: { repository: 'acme/app', labels: 'ready' } }, agents: ['claude', 'codex'], otherAgent: '', task: 'Add a test', workflow: 'traditional', step: 3 }; @@ -126,6 +144,26 @@ async function runTimed(timing: Timing, workflow: FactoryDraft['workflow'] = 'tr calls[calls.length - 1]!.limit = limit; return stopped ? 'timeout' : run.verdict; }; + // A detached branch check (RELAYFLOW_CHECK_WAIT): each call waits at most + // its wait, and the check runs until it finishes or reaches its total. + const spans = new Map(); + const polled = (command: string, lease?: string) => { + const id = /RELAYFLOW_CHECK_ID=([^;]+);/.exec(command)![1]!; + const wait = Number(/RELAYFLOW_CHECK_WAIT=(\d+);/.exec(command)![1]) / 60; + const limit = checkLimitMinutes(command); + const leaseMinutes = durationMs(lease)! / MINUTE; + leases.push({ name: 'check', limit: wait, lease: leaseMinutes }); + if (!spans.has(id)) spans.set(id, { run: timing.checks[Math.min(checkIndex++, timing.checks.length - 1)]!, ran: 0 }); + const span = spans.get(id)!; + const end = Math.min(span.run.minutes, limit); + const minutes = Math.min(wait, end - span.ran); + span.ran += minutes; + // With fullTimeouts each wait also holds its lease's minute of slack. + step('check', (minutes + (timing.fullTimeouts ? leaseMinutes - wait : 0)) * MINUTE, false, command); + calls[calls.length - 1]!.limit = limit; + if (span.ran < end) return 'running'; + return span.run.minutes > limit ? 'timeout' : span.run.verdict; + }; const originalError = console.error; console.error = (message: string) => { errors.push(String(message)); }; try { @@ -145,6 +183,7 @@ async function runTimed(timing: Timing, workflow: FactoryDraft['workflow'] = 'tr if (command.endsWith(FLOW_BASE_CHECK_COMMAND)) { return checked('base-check', command, timing.baseline ?? { minutes: FLOW_TIME.checkLimitMinutes, verdict: 'fail' }, runOptions?.timeout); } + if (command.endsWith(FLOW_CHECK_RUN_COMMAND) && command.includes('RELAYFLOW_CHECK_WAIT=')) return polled(command, runOptions?.timeout); if (command.endsWith(FLOW_CHECK_RUN_COMMAND)) { return checked('check', command, timing.checks[Math.min(checkIndex++, timing.checks.length - 1)]!, runOptions?.timeout); } @@ -184,16 +223,18 @@ const errorsOf = (run: Run) => run.errors.join('\n'); const after = (run: Run, name: string) => run.calls.slice(run.calls.findIndex(call => call.name === name) + 1); const named = (run: Run, prefix: string) => run.calls.filter(call => call.name.startsWith(prefix)); -describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { - it('plans for Cloud\'s 60-minute run cap', () => { +describe('Garden flow time plan (cloud#4235, cloud#4270, agentrelay.com#155)', () => { + it('plans within Cloud\'s 180-minute run cap', () => { const t = FLOW_TIME; - // Cloud caps a hosted run at 60 minutes, an E2B sandbox's lifetime, and - // that budget also pays for the sandbox setup before the body starts. - expect(t.headerMinutes).toBe(60); + // Cloud honours a header of up to 180 minutes, and that budget also pays + // for the sandbox setup before the body starts. 120 leaves an hour spare. + expect(t.headerMinutes).toBe(120); + expect(t.headerMinutes).toBeLessThanOrEqual(CLOUD_CAP_MINUTES); expect(t.bodyMinutes).toBe(t.headerMinutes - t.setupMinutes); expect(t.setupMinutes).toBeGreaterThanOrEqual(5); - // One repair round of about 15 minutes, one review of at most 20. - expect(t.repairMinutes).toBe(15); + // At most two repair rounds of 20 minutes, one review of at most 20. + expect(t.repairMinutes).toBe(20); + expect(t.repairRounds).toBe(2); expect(t.reviewMinutes).toBe(20); // A check stops itself before its 15-minute f.run lease. expect(t.checkLimitMinutes).toBeLessThan(t.checkMinutes); @@ -207,18 +248,141 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { expect(t.closeMinutes).toBeGreaterThanOrEqual(t.followUpMinutes + t.defaultStepMinutes); expect(t.publishMinutes).toBeGreaterThanOrEqual(t.forgeMinutes + t.pushMinutes + t.forgeMinutes + 4 * t.defaultStepMinutes + t.closeMinutes); expect(t.publishReserveMinutes).toBeGreaterThan(t.publishMinutes); - // Every agent limit is one relayflows accepts. + // Every agent limit is one relayflows accepts (AgentWorkforce/flows#606). + expect(t.agentLimitMaxMinutes).toBe(60); for (const minutes of [t.discoveryMinutes, t.repairMinutes, t.reviewMinutes, t.prototypeMinutes, t.comparatorMinutes]) { expect(minutes).toBeGreaterThan(0); expect(minutes).toBeLessThanOrEqual(t.agentLimitMaxMinutes); } // The steps before the implementer, at their limits, still leave it its - // floor and publishing: the mandatory path fits by construction. - expect(1 + t.discoveryMinutes + 3 + t.implementerFloorMinutes + t.publishReserveMinutes).toBeLessThanOrEqual(t.bodyMinutes); + // floor by the end of the build: the mandatory path fits by construction. + expect(1 + t.discoveryMinutes + 3 + t.implementerFloorMinutes).toBeLessThanOrEqual(t.buildByMinutes); + expect(t.buildReserveMinutes).toBe(t.bodyMinutes - t.buildByMinutes); + // What follows the build always holds a check at its whole budget, + // publishing and the review's floor. + expect(2 + t.checkTotalMinutes + t.publishReserveMinutes + t.reviewFloorMinutes + t.reviewReserveMinutes).toBeLessThanOrEqual(t.buildReserveMinutes); + // A check may span leases, each within f.run's 15 minutes. + expect(t.checkTotalMinutes).toBe(30); + expect(t.checkLimitMinutes + 1).toBeLessThanOrEqual(t.checkMinutes); + }); + + it('keeps the happy path\'s planned allowances at 109m, with the agents\' share where the 60-minute plan had it', () => { + // The extra time is margin, not a longer plan: the build ends where it + // did, and a clean run plans one check, publishing and one review. The + // check budget (14m to 30m, flows#626) and publishing's limits (17m to + // 25m) grew: 85m to 109m, +28%. Both are waits on a suite or a forge, and + // a clean run uses only what they take. The agents' allowances, the build + // window and the review, are pinned at the 60-minute plan's 54m. A change + // that lengthens the plan has to change this test. + const t = FLOW_TIME; + const planned = t.buildByMinutes + t.checkTotalMinutes + t.publishMinutes + t.reviewMinutes; + expect(planned).toBeLessThanOrEqual(PLANNED_CEILING_MINUTES); + expect(PLANNED_CEILING_MINUTES).toBe(109); + expect(SIXTY_MINUTE_PLAN_MINUTES).toBe(85); + expect(t.buildByMinutes + t.reviewMinutes).toBeLessThanOrEqual(34 + 20); + }); + + it('runs a 20-minute check across several leases and honours its verdict (flows#626)', async () => { + for (const verdict of ['pass', 'fail'] as const) { + const run = await runTimed({ + agents: { 'check-discovery': 6, 'implementer': 10, 'check-repair': 100, 'adversary': 8 }, + // A failing suite's re-check after the repair fails again, quickly. + checks: [{ minutes: 20, verdict }, { minutes: 2, verdict }], + baseline: { minutes: 2, verdict: 'fail' }, + reviewClean: true, + }); + const first = named(run, 'check').filter(call => call.name === 'check'); + const id = /RELAYFLOW_CHECK_ID=([^;]+);/.exec(first[0]!.command!)![1]; + const polls = first.filter(call => call.command!.includes(`RELAYFLOW_CHECK_ID=${id};`)); + // One check, waited on in more than one lease, each within f.run's 15 + // minutes, and run to the end: 20 minutes in all, not stopped at 14. + expect(polls.length, verdict).toBeGreaterThan(1); + expect(polls.reduce((sum, call) => sum + call.minutes, 0), verdict).toBeCloseTo(20); + for (const lease of run.leases.filter(lease => lease.name === 'check')) expect(lease.lease!).toBeLessThanOrEqual(FLOW_TIME.checkMinutes); + const report = run.calls.find(call => call.command?.includes('.relayflow/check-report.md'))!; + expect(report.command, verdict).toMatch(new RegExp(`^check=${verdict}; `)); + if (verdict === 'pass') expect(run.opened?.command).not.toContain(' --draft'); + else expect(run.opened?.command).toContain(' --draft'); + expect(run.refused, verdict).toBeNull(); + } + }); + + it('gives every branch check its ID, total and wait, and the base check only its limit', async () => { + const run = await runTimed({ + agents: { 'check-discovery': 1, 'implementer': 1, 'check-repair': 1 }, + checks: [{ minutes: 20, verdict: 'fail' }, { minutes: 2, verdict: 'fail' }], + baseline: { minutes: 2, verdict: 'fail' }, + }, 'simple'); + const checks = run.calls.filter(call => call.name === 'check'); + expect(checks.length).toBeGreaterThan(2); + const ids = new Set(); + for (const call of checks) { + const match = /^RELAYFLOW_CHECK_ID=(\d+-\d+); RELAYFLOW_CHECK_TIMEOUT=(\d+); RELAYFLOW_CHECK_WAIT=(\d+); /.exec(call.command!); + expect(match, call.command).not.toBeNull(); + ids.add(match![1]!); + expect(Number(match![2]) / 60).toBeLessThanOrEqual(FLOW_TIME.checkTotalMinutes); + expect(Number(match![3]) / 60).toBeLessThanOrEqual(FLOW_TIME.checkLimitMinutes); + } + // The first check and the re-checks after the two repairs: one ID each. + expect(ids.size).toBe(3); + const base = run.calls.find(call => call.name === 'base-check')!; + expect(base.command).toMatch(/^base=abc123; RELAYFLOW_CHECK_TIMEOUT=\d+; /); + expect(base.command).not.toContain('RELAYFLOW_CHECK_WAIT='); + }); + + it('keeps a local run\'s single 15-minute repair, with no second round', async () => { + const run = await runTimed({ + agents: { 'check-repair': 5 }, + checks: [{ minutes: 2, verdict: 'fail' }], + baseline: { minutes: 2, verdict: 'fail' }, + }, 'traditional', { target: 'local' }); + expect(named(run, 'check-repair').map(call => call.name)).toEqual(['check-repair']); + expect(factorySource({ ...draft, workflow: 'simple' }, 'local')).toContain('round <= 1 &&'); + expect(factorySource({ ...draft, workflow: 'simple' }, 'cloud')).toContain('round <= 2 &&'); + expect(FLOW_TIME.localRepairMinutes).toBe(15); + expect(FLOW_TIME.localRepairRounds).toBe(1); }); - it('declares the 1h wallclock in the cloud header, and keeps 3h for a local run', () => { - expect(factorySource(draft)).toContain('{ budget: { wallclock: "1h" } }'); + it('drafts, reviews and repairs nothing when the checks could not be run at all', async () => { + // No limiter on the machine, so the command refuses to start the suite + // (unrunnable). That is not a verdict on the change: nothing to repair, + // no base commit to compare with, and the review still runs. + const run = await runTimed({ + agents: { 'check-discovery': 6, 'implementer': 10, 'adversary': 8 }, + checks: [{ minutes: 1, verdict: 'unrunnable' }], + reviewClean: true, + }); + expect(named(run, 'check-repair')).toHaveLength(0); + expect(run.names).not.toContain('base-check'); + expect(run.calls.find(call => call.command?.includes('.relayflow/check-report.md'))?.command).toMatch(/^check=unrunnable; baseline=; /); + expect(run.opened?.command).toContain(' --draft'); + expect(errorsOf(run)).toMatch(/The checks could not be run/); + expect(run.names).toContain('adversary'); + expect(run.finish).toBe('needs_human'); + expect(run.refused).toBeNull(); + }); + + it('reports a check that runs past its whole budget as a timeout, not a failure', async () => { + const run = await runTimed({ + agents: { 'check-discovery': 6, 'implementer': 10, 'check-repair': 100 }, + checks: [{ minutes: 45, verdict: 'pass' }], + }, 'simple'); + const first = run.calls.filter(call => call.name === 'check').slice(0, 1)[0]!; + expect(first.limit).toBe(FLOW_TIME.checkTotalMinutes); + const report = run.calls.find(call => call.command?.includes('.relayflow/check-report.md'))!; + expect(report.command).toMatch(/^check=timeout; /); + expect(errorsOf(run)).toMatch(/The checks timed out \(stopped at their time budget, not a test failure\)/); + expect(errorsOf(run)).not.toMatch(/The checks fail\b/); + expect(run.opened?.command).toContain(' --draft'); + expect(run.refused).toBeNull(); + }); + + it('declares the 2h wallclock in the cloud header, and keeps 3h for a local run', () => { + for (const workflow of ['traditional', 'prototype', 'simple'] as const) { + const source = factorySource({ ...draft, workflow }); + expect(source).toContain('{ budget: { wallclock: "2h" } }'); + expect(declaredWallclockMs(source)).toBeLessThanOrEqual(CLOUD_CAP_MINUTES * MINUTE); + } // A local run has no sandbox lifetime, and its pinned runtime cannot stop // an agent at a limit, so it keeps the longer budget. expect(factorySource(draft, 'local')).toContain('{ budget: { wallclock: "3h" } }'); @@ -271,7 +435,7 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { // charged exactly, its limit (agentrelay.com#138). Sweeping the // implementer moves the run across each guard's edge: wherever a check, // repair, base check or review only just starts, what follows it must - // still fit, and the worst case stays inside Cloud's 60 minutes. + // still fit, and the worst case stays inside the header's 120 minutes. const seen: Record> = { traditional: new Set(), prototype: new Set(), simple: new Set() }; let checked = 0; let timedOut = 0; @@ -283,8 +447,9 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { // Prototypes that run to their limit take everything above the // implementer's floor, so with a fast suite they (and the // comparator) finish early, which leaves the base check something - // to guard. - agents: { 'check-discovery': early, 'prototype': checkMinutes === 3 ? 2 : 100, 'comparator': checkMinutes === 3 ? 1 : 100, 'implementer': implementer, 'check-repair': 100, 'adversary': 100 }, + // to guard. A repair that finishes before its limit, on a fast + // suite that still fails, leads to the second round. + agents: { 'check-discovery': early, 'prototype': checkMinutes === 3 ? 2 : 100, 'comparator': checkMinutes === 3 ? 1 : 100, 'implementer': implementer, 'check-repair': checkMinutes === 3 ? 2 : 100, 'adversary': 100 }, checks: [{ minutes: checkMinutes, verdict }], baseline: { minutes: 100, verdict: 'fail' }, fullTimeouts: true, @@ -298,7 +463,7 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { // steps before the flow first reads its clock (the working-file // exclude and that read), charged here at 30s each. expect(run.chargedMinutes, where).toBeLessThanOrEqual(FLOW_TIME.bodyMinutes + 2 * FLOW_TIME.defaultStepMinutes); - expect(FLOW_TIME.setupMinutes + run.chargedMinutes, where).toBeLessThanOrEqual(60 + 2 * FLOW_TIME.defaultStepMinutes); + expect(FLOW_TIME.setupMinutes + run.chargedMinutes, where).toBeLessThanOrEqual(FLOW_TIME.headerMinutes + 2 * FLOW_TIME.defaultStepMinutes); for (const call of run.calls) { // Every agent states a limit on the cloud target, within the runtime's ceiling. if (call.name === 'run' || call.name === 'check' || call.name === 'base-check' || call.command !== undefined) continue; @@ -307,51 +472,111 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { expect(call.limit!, `${where}: ${call.name}`).toBeLessThanOrEqual(FLOW_TIME.agentLimitMaxMinutes); if (call.timedOut) timedOut++; } - for (const name of run.names) seen[workflow]!.add(name.startsWith('check-repair') ? 'check-repair' : name); + for (const name of run.names) seen[workflow]!.add(name.startsWith('prototype') ? 'prototype' : name); if (run.errors.some(error => /skipped the checks/i.test(error))) seen[workflow]!.add('skipped-checks'); } } } - // Not vacuous: every workflow reaches each guarded optional step and the - // fail-fast path somewhere in the sweep. The prototype workflow never - // affords a repair here: three parallel prototypes are charged three - // times, and with the comparator they use what a repair would need. + // Not vacuous: every workflow reaches each guarded optional step somewhere + // in the sweep. The build ends by buildByMinutes, so the checks always run + // on Cloud now: the fail-fast path that skips them is never taken. expect(checked).toBeGreaterThan(1000); expect(timedOut).toBeGreaterThan(1000); for (const workflow of ['traditional', 'prototype', 'simple']) { - for (const name of ['check', 'check-repair', 'base-check', 'free-disk', 'skipped-checks', 'time-stop']) { - if (workflow === 'simple' && name === 'time-stop') continue; - if (workflow === 'prototype' && name === 'check-repair') continue; + for (const name of ['check', 'check-repair', 'check-repair-2', 'base-check', 'free-disk']) { expect(seen[workflow], `${workflow} reaches ${name}`).toContain(name); } + expect(seen[workflow], `${workflow} never skips the checks`).not.toContain('skipped-checks'); + // Every step after the build leaves the review its floor, so a review + // is never stopped for time on Cloud: the time-stop path stays only as + // a guard. + expect(seen[workflow], `${workflow} never stops for time before the review`).not.toContain('time-stop'); } expect(seen.traditional).toContain('adversary'); expect(seen.prototype).toContain('adversary'); }, 60_000); // About a thousand simulated runs. - it('runs at most one repair round', async () => { - const run = await runTimed({ + it('runs at most two repair rounds, the second only when the re-check still fails', async () => { + const failing = await runTimed({ agents: {}, checks: [{ minutes: 2, verdict: 'fail' }], baseline: { minutes: 2, verdict: 'fail' }, }, 'simple'); + expect(named(failing, 'check-repair').map(call => call.name)).toEqual(['check-repair', 'check-repair-2']); + for (const repair of named(failing, 'check-repair')) expect(repair.limit).toBeLessThanOrEqual(FLOW_TIME.repairMinutes); + // The first check, a re-check after each repair, and the base commit. + expect(failing.names.filter(name => name === 'check')).toHaveLength(3); + expect(failing.names).toContain('base-check'); + expect(failing.opened?.command).toContain(' --draft'); + // A first repair that passes the re-check ends the repairs. + const fixed = await runTimed({ + agents: {}, + checks: [{ minutes: 2, verdict: 'fail' }, { minutes: 2, verdict: 'pass' }], + }, 'simple'); + expect(named(fixed, 'check-repair')).toHaveLength(1); + // A clean run repairs nothing: the second round is never planned. + const clean = await runTimed({ agents: {}, checks: [{ minutes: 2, verdict: 'pass' }], reviewClean: true }); + expect(named(clean, 'check-repair')).toHaveLength(0); + }); + + it('gives a failed check the extra repair room, so a slow repair finishes instead of being stopped', async () => { + // bda21b91's first repair took 19m27s against an 11m suite: stopped at 15 + // under the 60-minute plan, it now finishes, and the pull request is ready. + const slow = await runTimed({ + agents: { 'check-discovery': 6, 'implementer': 10, 'check-repair': 19.5, 'adversary': 8 }, + checks: [{ minutes: 11.5, verdict: 'fail' }, { minutes: 11.4, verdict: 'pass' }], + reviewClean: true, + }); + expect(named(slow, 'check-repair')).toHaveLength(1); + expect(named(slow, 'check-repair')[0]!.timedOut).toBeUndefined(); + expect(slow.opened?.command).not.toContain(' --draft'); + expect(slow.names).toContain('adversary'); + expect(slow.refused).toBeNull(); + // With a faster suite, checks that still fail get a second round, and + // the review still runs after it: the second round leaves it its floor, + // and the base check its time. + const twice = await runTimed({ + agents: { 'check-discovery': 6, 'implementer': 10, 'check-repair-2': 6, 'check-repair': 19.5, 'adversary': 8 }, + checks: [{ minutes: 3, verdict: 'fail' }, { minutes: 3, verdict: 'fail' }, { minutes: 3, verdict: 'pass' }], + reviewClean: true, + }); + const repairs = named(twice, 'check-repair'); + expect(repairs.map(call => call.name)).toEqual(['check-repair', 'check-repair-2']); + for (const repair of repairs) expect(repair.timedOut).toBeUndefined(); + expect(twice.opened?.command).not.toContain(' --draft'); + expect(twice.names).toContain('adversary'); + expect(twice.names).not.toContain('time-stop'); + expect(twice.refused).toBeNull(); + expect(FLOW_TIME.setupMinutes + twice.chargedMinutes).toBeLessThanOrEqual(FLOW_TIME.headerMinutes); + }); + + it('never runs a second repair after one that was stopped at its limit', async () => { + // Another repair of the same failures would most likely run out too. + const run = await runTimed({ + agents: { 'implementer': 100, 'check-repair': 100 }, + checks: [{ minutes: FLOW_TIME.checkLimitMinutes, verdict: 'fail' }], + baseline: { minutes: 2, verdict: 'fail' }, + }); + expect(run.refused).toBeNull(); expect(named(run, 'check-repair')).toHaveLength(1); - expect(named(run, 'check-repair')[0]!.limit).toBeLessThanOrEqual(FLOW_TIME.repairMinutes); - // The first check, the re-check after the repair, and the base commit. - expect(run.names.filter(name => name === 'check')).toHaveLength(2); - expect(run.names).toContain('base-check'); + expect(named(run, 'check-repair')[0]!.timedOut).toBe(true); + expect(errorsOf(run)).toContain('no further repair is tried'); expect(run.opened?.command).toContain(' --draft'); + expect(run.names).toContain('adversary'); }); - it('skips a repair it cannot afford and publishes the draft with time to spare', async () => { + it('skips a second repair that would cost the review its floor, and still publishes and reviews', async () => { + // A 14m suite after a 20m implementer leaves too little for another round + // that still leaves the review its floor. const run = await runTimed({ - agents: { 'implementer': 15, 'check-repair': FLOW_TIME.repairMinutes }, + agents: { 'implementer': 20, 'check-repair': 12 }, checks: [{ minutes: FLOW_TIME.checkLimitMinutes, verdict: 'fail' }], - }, 'simple'); + baseline: { minutes: 2, verdict: 'fail' }, + }); + expect(named(run, 'check-repair')).toHaveLength(1); + expect(errorsOf(run)).toMatch(/skipped the second repair/i); + expect(run.names).toContain('adversary'); expect(run.refused).toBeNull(); - expect(named(run, 'check-repair')).toHaveLength(0); - expect(run.opened?.command).toContain(' --draft'); - expect(errorsOf(run)).toMatch(/skipped the repair/i); }); it('expects the re-check to take about as long as the first check, not a fresh full run', async () => { @@ -372,7 +597,7 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { baseline: { minutes: 2, verdict: 'fail' }, }, 'simple'); const checks = run.calls.map((call, index) => ({ call, index })).filter(({ call }) => call.name === 'check' || call.name === 'base-check'); - expect(checks).toHaveLength(3); + expect(checks).toHaveLength(4); expect(run.calls[checks[0]!.index - 1]?.name).not.toBe('free-disk'); for (const { index } of checks.slice(1)) expect(run.calls.slice(checks[0]!.index, index).map(call => call.name)).toContain('free-disk'); }); @@ -434,22 +659,22 @@ describe('Garden flow time plan (cloud#4235, agentrelay.com#155)', () => { expect(errorsOf(run)).not.toMatch(/as far as the base commit shows/); }); - it('fails fast to a draft when there is no time left to run the checks', async () => { - // The implementer spends everything it is given: it is stopped at its - // limit, which leaves room only to publish, at every step's limit. + it('stops a slow implementer where the 60-minute plan did, and still checks, publishes and reviews', async () => { + // The implementer spends everything it is given: it is stopped by minute + // buildByMinutes, and what follows has room for the checks, publishing at + // every step's limit, and the review, instead of skipping the checks. const run = await runTimed({ agents: { 'implementer': 100 }, checks: [{ minutes: 5, verdict: 'pass' }], fullTimeouts: true }); const implementer = run.calls.find(call => call.name === 'implementer')!; expect(implementer.timedOut).toBe(true); - expect(run.names).not.toContain('check'); - expect(errorsOf(run)).toMatch(/skipped the checks/i); + // The clock starts after the two steps before its first read. + expect(implementer.atMinute + implementer.minutes).toBeLessThanOrEqual(FLOW_TIME.buildByMinutes + 2 * FLOW_TIME.defaultStepMinutes); expect(errorsOf(run)).toMatch(/implementer was stopped at its \d+m limit/); + expect(errorsOf(run)).not.toMatch(/skipped the checks/i); const report = run.calls.find(call => call.command?.includes('.relayflow/check-report.md')); - expect(report?.command).toMatch(/^check=skipped; .*implementer_timeout=yes; /); + expect(report?.command).toMatch(/^check=pass; .*implementer_timeout=yes; /); expect(run.opened?.command).toContain(' --draft'); - // The review cannot fit either: the existing time-stop path drafts and says so. - expect(run.names).toContain('time-stop'); - expect(run.names).not.toContain('adversary'); - expect(run.finish).toBe('needs_human'); + expect(run.names).toContain('adversary'); + expect(run.names).not.toContain('time-stop'); expect(run.refused).toBeNull(); }); diff --git a/web/lib/test/flow-local.test.ts b/web/lib/test/flow-local.test.ts index 6db7ea34..e2892dec 100644 --- a/web/lib/test/flow-local.test.ts +++ b/web/lib/test/flow-local.test.ts @@ -17,8 +17,12 @@ import { FLOW_BASE_CHECK_COMMAND, FLOW_CHECK_RUN_COMMAND, FLOW_REVIEW_BLOCKED_CO */ // Every push goes through the workflow-file guard (run 065fd98f). const PUSH = 'base=; ' + FLOW_PUSH_COMMAND + ' --set-upstream origin HEAD'; -/** A check command without the limit the flow gives it (RELAYFLOW_CHECK_TIMEOUT), so it compares as written. */ -const plain = (command: string) => command.replace(/RELAYFLOW_CHECK_TIMEOUT=\d+; /, ''); +/** + * A check command without what the flow gives it, so it compares as written: + * a branch check's ID, total and wait, in that order, or the base check's + * limit alone. A branch check missing any of the three no longer matches. + */ +const plain = (command: string) => command.replace(/^RELAYFLOW_CHECK_ID=\d+-\d+; RELAYFLOW_CHECK_TIMEOUT=\d+; RELAYFLOW_CHECK_WAIT=\d+; /, '').replace(/^(base=[^;]*; )RELAYFLOW_CHECK_TIMEOUT=\d+; /, '$1'); function answer(raw: string, { publish = 'publish', clean = 'yes', check = 'pass' as string | (() => string), baseline = 'pass' } = {}) { const command = plain(raw); diff --git a/web/lib/test/flow-onboarding.test.ts b/web/lib/test/flow-onboarding.test.ts index f7aba42b..8d2e229a 100644 --- a/web/lib/test/flow-onboarding.test.ts +++ b/web/lib/test/flow-onboarding.test.ts @@ -11,8 +11,12 @@ import { FLOW_PUSH_COMMAND, FLOW_REFERENCE_LIMIT } from '../flow-workflows'; // Every push goes through the workflow-file guard (run 065fd98f). const PUSH = 'base=abc123; ' + FLOW_PUSH_COMMAND + ' --set-upstream origin HEAD'; -/** A check command without the limit the flow gives it (RELAYFLOW_CHECK_TIMEOUT), so it compares as written. */ -const plain = (command: string) => command.replace(/RELAYFLOW_CHECK_TIMEOUT=\d+; /, ''); +/** + * A check command without what the flow gives it, so it compares as written: + * a branch check's ID, total and wait, in that order, or the base check's + * limit alone. A branch check missing any of the three no longer matches. + */ +const plain = (command: string) => command.replace(/^RELAYFLOW_CHECK_ID=\d+-\d+; RELAYFLOW_CHECK_TIMEOUT=\d+; RELAYFLOW_CHECK_WAIT=\d+; /, '').replace(/^(base=[^;]*; )RELAYFLOW_CHECK_TIMEOUT=\d+; /, '$1'); const matchingIssue = { source: 'github', title: ' Fix login ', body: 'Login fails', labels: ['ready', 'bug'], repository: 'acme/app', identifier: '#507', url: 'https://github.com/acme/app/issues/507' }; @@ -101,7 +105,7 @@ describe('software factory onboarding', () => { const again = source.indexOf('if (checkPlan === "none") await f.run(resolveChecks);'); expect(implementer, workflow).toBeGreaterThan(-1); expect(again, workflow).toBeGreaterThan(implementer); - expect(source.indexOf('await timedCheck("", runChecks'), workflow).toBeGreaterThan(again); + expect(source.indexOf('await spannedCheck(checkLimit)'), workflow).toBeGreaterThan(again); } }); @@ -249,8 +253,8 @@ describe('software factory onboarding', () => { it('gives Cloud flows a wall-clock budget so unpriced agents are never refused', () => { for (const agents of [['claude', 'codex'], ['codex'], ['claude']] as FactoryDraft['agents'][]) { const source = factorySource({ ...completed, agents }); - // Cloud's cap on a hosted run (AgentWorkforce/cloud#4235). - expect(source).toContain('{ budget: { wallclock: "1h" } }'); + // An hour under Cloud's 180-minute cap on a hosted run (AgentWorkforce/cloud#4270). + expect(source).toContain('{ budget: { wallclock: "2h" } }'); expect(source).not.toMatch(/budget: "\$\d/); } }); @@ -532,8 +536,9 @@ describe('software factory onboarding', () => { it('opens a draft and keeps reviewing when the base commit fails the same checks', async () => { // 139d1a46's shape: the failure is the environment's, not the change's. const { calls, finish, errors } = await runFactory([true, true], true, matchingIssue, completed, 'publish', ['fail', 'fail', 'fail'], 'fail'); - // One repair round, then the base commit (agentrelay.com#155). - expect(calls.filter(call => call.startsWith('check-repair'))).toHaveLength(1); + // Two repair rounds, the second only because the re-check still fails, + // then the base commit (agentrelay.com#155). + expect(calls.filter(call => call.startsWith('check-repair'))).toEqual(['check-repair:claude', 'check-repair-2:claude']); expect(calls.find(call => call.endsWith(FLOW_BASE_CHECK_COMMAND))).toBe('base=abc123; ' + FLOW_BASE_CHECK_COMMAND); expect(reportCall(calls)).toMatch(/^check=fail; baseline=fail; /); expect(createCall(calls)).toContain('--draft'); diff --git a/web/lib/test/flow-workflows.test.ts b/web/lib/test/flow-workflows.test.ts index d6c04ec9..d2bf27c8 100644 --- a/web/lib/test/flow-workflows.test.ts +++ b/web/lib/test/flow-workflows.test.ts @@ -158,7 +158,7 @@ describe('FLOW_CHECK_RUN_COMMAND', () => { // Only the tools the command needs, plus perl: no timeout, no gtimeout. const bin = fixture({}); const root0 = () => fixture({}); - for (const tool of ['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'perl']) { + for (const tool of ['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'perl', 'rm', 'date', 'mv']) { const found = spawnSync('/bin/sh', ['-c', `command -v ${tool}`], { encoding: 'utf8' }).stdout.trim(); if (found) symlinkSync(found, path.join(bin, tool)); } @@ -203,6 +203,210 @@ describe('FLOW_CHECK_RUN_COMMAND', () => { expect(token).toBe('timeout'); }); + describe('across several leases (flows#626)', () => { + /** Calls the command as the flow does, one bounded wait after another, until it stops saying `running`. */ + function spanned(root: string, id: string, total: number, wait = 1) { + const tokens: string[] = []; + const started = Date.now(); + let result = { code: 0 as number | null, token: 'running', stdout: '', stderr: '' }; + while (result.token === 'running' && Date.now() - started < 60_000) { + const call = Date.now(); + result = sh(FLOW_CHECK_RUN_COMMAND, root, { RELAYFLOW_CHECK_ID: id, RELAYFLOW_CHECK_TIMEOUT: String(total), RELAYFLOW_CHECK_WAIT: String(wait) }); + // Each call returns within its wait (with a second of polling, and the + // 5s grace before SIGKILL when it stops a check), never holding the + // lease for the whole check. + expect(Date.now() - call).toBeLessThan((wait + 8) * 1000); + expect(result.code).toBe(0); + tokens.push(result.token); + } + return { tokens, last: result, exit: read(root, '.relayflow/check.log.exit').trim(), elapsed: Number(read(root, '.relayflow/check.log.elapsed').trim()), limit: Number(read(root, '.relayflow/check.log.limit').trim()), log: read(root, '.relayflow/check.log') }; + } + + /** Whether `pid` (a process, or with a minus a group) is gone, allowing a moment for the kernel to reap it. */ + function gone(pid: number) { + const until = Date.now() + 3000; + for (;;) { + try { process.kill(pid, 0); } catch { return true; } + if (Date.now() > until) return false; + spawnSync('/bin/sleep', ['0.1']); + } + } + + it('finishes a check longer than one wait across calls, and honours its exit status', () => { + const passing = spanned(fixture({ [FLOW_CHECK_SCRIPT]: 'echo building\nsleep 3\necho suite-ran\n' }), 'a-1', 60); + expect(passing.tokens.length).toBeGreaterThan(1); + expect(passing.tokens.slice(0, -1).every(token => token === 'running')).toBe(true); + expect(passing.tokens.at(-1)).toBe('pass'); + expect(passing.exit).toBe('0'); + expect(passing.log).toContain('suite-ran'); + + const failing = spanned(fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 3\necho broke\nexit 3\n' }), 'a-1', 60); + expect(failing.tokens.length).toBeGreaterThan(1); + expect(failing.tokens.at(-1)).toBe('fail'); + expect(failing.exit).toBe('3'); + expect(failing.last.stderr).toContain('exit 3'); + expect(failing.last.stderr).toContain('broke'); + }, 30_000); + + it.skipIf(spawnSync('/bin/sh', ['-c', 'command -v timeout || command -v gtimeout || command -v perl']).status !== 0)('reports a check that runs past its total as a timeout, not a failure', () => { + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 30\n' }); + const stopped = spanned(root, 'b-1', 2); + expect(stopped.tokens.at(-1)).toBe('timeout'); + expect(stopped.exit).toBe('124'); + expect(stopped.limit).toBe(2); + expect(stopped.elapsed).toBeGreaterThanOrEqual(2); + expect(stopped.elapsed).toBeLessThan(10); + }, 30_000); + + it('stops the check\'s whole process group at its total, with SIGKILL after a grace when SIGTERM is ignored', () => { + // The suite starts a child of its own and ignores SIGTERM: only a + // group-wide SIGKILL stops it all. + const root = fixture({ [FLOW_CHECK_SCRIPT]: "trap '' TERM\nsleep 60 &\necho $! > .relayflow/child.pid\nwhile :; do sleep 1; done\n" }); + const stopped = spanned(root, 'd-1', 2); + expect(stopped.tokens.at(-1)).toBe('timeout'); + expect(stopped.log).toContain('stopped the checks after'); + const group = Number(read(root, '.relayflow/check.log.group').trim()); + const child = Number(read(root, '.relayflow/child.pid').trim()); + expect(group).toBeGreaterThan(0); + expect(child).toBeGreaterThan(0); + expect(gone(group)).toBe(true); + expect(gone(child)).toBe(true); + expect(gone(-group)).toBe(true); + }, 30_000); + + it('reports a suite\'s own exit 124 as a failure, not as the flow\'s timeout', () => { + // A CI-shaped check.sh can run `timeout 60 some-test` itself. + const own = spanned(fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 2\nexit 124\n' }), 'f-1', 60); + expect(own.tokens.at(-1)).toBe('fail'); + expect(own.exit).toBe('124'); + }, 30_000); + + it('tells a resumed waiter that a check stopped for time timed out', () => { + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 30\n' }); + expect(spanned(root, 'g-1', 2).tokens.at(-1)).toBe('timeout'); + // The same ID again, as a resumed run asks: still a timeout, not a fail. + const again = sh(FLOW_CHECK_RUN_COMMAND, root, { RELAYFLOW_CHECK_ID: 'g-1', RELAYFLOW_CHECK_TIMEOUT: '2', RELAYFLOW_CHECK_WAIT: '1' }); + expect(again.token).toBe('timeout'); + }, 30_000); + + it('stops a check an earlier attempt left running before it starts a new one', () => { + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'echo $$ >> .relayflow/suites\nsleep 60\n' }); + // The first attempt starts and is abandoned while its check runs. + expect(sh(FLOW_CHECK_RUN_COMMAND, root, { RELAYFLOW_CHECK_ID: 'h-1', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '1' }).token).toBe('running'); + const first = Number(read(root, '.relayflow/check.log.group').trim()); + expect(gone(-first)).toBe(false); + const next = sh(FLOW_CHECK_RUN_COMMAND, root, { RELAYFLOW_CHECK_ID: 'h-2', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '1' }); + expect(next.token).toBe('running'); + expect(next.stderr).toContain('stopped a check an earlier attempt left running'); + expect(gone(-first)).toBe(true); + // Where there is no ps to identify it, a live group with no status is + // stopped anyway rather than left writing the same files. + const noPs = onlyTools(['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'rm', 'date', 'mv', 'setsid', 'perl']); + const other = fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 60\n' }); + expect(sh(FLOW_CHECK_RUN_COMMAND, other, { PATH: noPs, RELAYFLOW_CHECK_ID: 'j-1', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '1' }).token).toBe('running'); + const left = Number(read(other, '.relayflow/check.log.group').trim()); + const after = sh(FLOW_CHECK_RUN_COMMAND, other, { PATH: noPs, RELAYFLOW_CHECK_ID: 'j-2', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '1' }); + expect(after.stderr).toContain('no ps here to identify it'); + expect(gone(-left)).toBe(true); + try { process.kill(-Number(read(other, '.relayflow/check.log.group').trim()), 'SIGKILL'); } catch { /* already gone */ } + // Clean up the second check. + const second = Number(read(root, '.relayflow/check.log.group').trim()); + try { process.kill(-second, 'SIGKILL'); } catch { /* already gone */ } + }, 30_000); + + /** A PATH with only `tools` on it, so a missing one is genuinely missing. */ + function onlyTools(tools: string[]) { + const bin = fixture({}); + for (const tool of tools) { + const found = spawnSync('/bin/sh', ['-c', `command -v ${tool}`], { encoding: 'utf8' }).stdout.trim(); + if (found) symlinkSync(found, path.join(bin, tool)); + } + return bin; + } + + it.skipIf(spawnSync('/bin/sh', ['-c', 'command -v perl']).status !== 0)('puts the check in a group of its own where there is no setsid, as on macOS', () => { + const bin = onlyTools(['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'perl', 'rm', 'date', 'mv', 'ps', 'grep']); + expect(sh('command -v setsid || echo none', fixture({}), { PATH: bin }).stdout.trim()).toBe('none'); + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 60 &\necho $! > .relayflow/child.pid\nwait\n' }); + let result = { token: 'running', code: 0 as number | null, stdout: '', stderr: '' }; + for (let i = 0; i < 20 && result.token === 'running'; i++) { + result = sh(FLOW_CHECK_RUN_COMMAND, root, { PATH: bin, RELAYFLOW_CHECK_ID: 'e-1', RELAYFLOW_CHECK_TIMEOUT: '2', RELAYFLOW_CHECK_WAIT: '1' }); + } + expect(result.token).toBe('timeout'); + const group = Number(read(root, '.relayflow/check.log.group').trim()); + // Its own group: not this test's, and gone with everything in it. + expect(group).not.toBe(process.pid); + expect(gone(-group)).toBe(true); + expect(gone(Number(read(root, '.relayflow/child.pid').trim()))).toBe(true); + }, 30_000); + + it('runs the check in the step itself where nothing can put it in a group of its own', () => { + // Without setsid and perl nothing could stop a detached check's + // descendants, so it is never detached: it runs here, within the wait, + // where the limiter stops it. macOS has no timeout or gtimeout, and perl + // is excluded here on purpose, so the fixture brings its own limiter: + // without one the command refuses to run the checks at all (below), and + // the timeout case would never exercise the timeout path. + const bin = onlyTools(['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'rm', 'date', 'mv']); + writeFileSync(path.join(bin, 'timeout'), '#!/bin/sh\nsecs=$1; shift\n"$@" & c=$!\n( sleep "$secs"; kill -TERM "$c" 2>/dev/null ) & w=$!\nwait "$c"; status=$?\nkill -TERM "$w" 2>/dev/null\nif [ "$status" -ge 128 ]; then exit 124; fi\nexit "$status"\n', { mode: 0o755 }); + expect(sh('command -v setsid || command -v perl || echo none', fixture({}), { PATH: bin }).stdout.trim()).toBe('none'); + expect(sh('command -v timeout', fixture({}), { PATH: bin }).stdout.trim()).toBe(path.join(bin, 'timeout')); + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'echo suite-ran\n' }); + const passing = sh(FLOW_CHECK_RUN_COMMAND, root, { PATH: bin, RELAYFLOW_CHECK_ID: 'i-1', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '5' }); + expect(passing.token).toBe('pass'); + expect(passing.stderr).toContain('neither setsid nor perl'); + // Nothing was detached, so there is no group to wait on, and the step + // cannot hold its lease longer than the wait it was given. + expect(read(root, '.relayflow/check.log.group')).toBe(''); + expect(read(root, '.relayflow/check.log')).toContain('suite-ran'); + // The wait, not the total, is what bounds this step: a 30s suite with a + // 60s total and a 2s wait is stopped at 2s and reported as a timeout. + const slow = fixture({ [FLOW_CHECK_SCRIPT]: 'sleep 30\n' }); + const started = Date.now(); + const timedOut = sh(FLOW_CHECK_RUN_COMMAND, slow, { PATH: bin, RELAYFLOW_CHECK_ID: 'i-2', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '2' }); + expect(timedOut.token).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(20_000); + // The limit it was held to, and reports, is the wait, not the total. + expect(read(slow, '.relayflow/check.log.limit').trim()).toBe('2'); + }, 30_000); + + it('refuses to start the checks where nothing can stop them at a time limit', () => { + // No timeout, no gtimeout, no perl: a suite started here could not be + // stopped, would hold the step's lease until the runner killed it, and + // the run would end with no verdict. Cloud's image always has coreutils + // `timeout`, so only a local run can reach this. + const bin = onlyTools(['sh', 'dirname', 'mkdir', 'tail', 'cat', 'sleep', 'rm', 'date', 'mv', 'setsid']); + expect(sh('command -v timeout || command -v gtimeout || command -v perl || echo none', fixture({}), { PATH: bin }).stdout.trim()).toBe('none'); + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'echo suite-ran > ran.txt\nsleep 30\n' }); + // Both paths refuse: a branch check waited on across leases, and the + // single-call check the base-commit comparison uses. + const cases: Record[] = [{ RELAYFLOW_CHECK_ID: 'k-1', RELAYFLOW_CHECK_TIMEOUT: '60', RELAYFLOW_CHECK_WAIT: '2' }, { RELAYFLOW_CHECK_TIMEOUT: '60' }]; + for (const env of cases) { + const started = Date.now(); + const result = sh(FLOW_CHECK_RUN_COMMAND, root, { PATH: bin, ...env }); + // Bounded, explicit, and exit 0 like every other verdict. + expect(result).toMatchObject({ code: 0, token: 'unrunnable' }); + expect(Date.now() - started).toBeLessThan(10_000); + expect(result.stderr).toContain('no timeout, gtimeout or perl'); + expect(result.stderr).toContain('Install coreutils (timeout) or perl'); + expect(read(root, '.relayflow/check.log')).toContain('they were not run'); + // The suite never started, so nothing was detached and nothing ran. + expect(read(root, 'ran.txt')).toBe(''); + expect(read(root, '.relayflow/check.log.group')).toBe(''); + } + }, 30_000); + + it('waits for the check an ID already started instead of starting another, so a resumed run picks it up', () => { + const root = fixture({ [FLOW_CHECK_SCRIPT]: 'echo started >> .relayflow/starts\nsleep 3\n' }); + const first = spanned(root, 'c-1', 60); + expect(first.tokens.at(-1)).toBe('pass'); + expect(read(root, '.relayflow/starts').trim().split('\n')).toHaveLength(1); + // A new ID is a new check: the re-check after a repair. + spanned(root, 'c-2', 60); + expect(read(root, '.relayflow/starts').trim().split('\n')).toHaveLength(2); + }, 30_000); + }); + it('runs the resolved default end to end, and still fails a failing suite', () => { const run = (files: Record, env: Record = {}) => { const root = fixture(files); @@ -715,6 +919,32 @@ describe('FLOW_CHECK_REPORT_COMMAND', () => { expect(unchecked.body).not.toContain('could not be checked for comparison'); }); + it('tells a failed check, a timed-out check and checks that never ran apart', () => { + // A failure: its exit status, and the end of its output. + const failed = report('fail', 'skipped', { '.relayflow/check.log.exit': '3\n', '.relayflow/check.log.elapsed': '200\n', '.relayflow/check.log.limit': '1800\n' }); + expect(failed.body).toContain('**The checks failed** (exit 3). The end of their output is below.'); + expect(failed.body).toContain('FAIL src/login.test.ts'); + expect(failed.body).not.toContain('timed out'); + // A timeout (flows#626): how long it ran against its budget, and what to + // do about it. Never "The checks failed". + const timedOut = report('timeout', 'skipped', { '.relayflow/check.log.exit': '124\n', '.relayflow/check.log.elapsed': '1803\n', '.relayflow/check.log.limit': '1800\n' }); + expect(timedOut.body).toContain('**The checks timed out**: they ran for 30m03s of their 30m00s budget and were stopped before they finished, so this is a time limit, not a test failure.'); + expect(timedOut.body).toContain('Run them locally (`sh .relayflow/check.sh`)'); + expect(timedOut.body).toContain('raise the check budget (`checkTotal` in the flow)'); + expect(timedOut.body).toContain('FAIL src/login.test.ts'); + expect(timedOut.body).not.toContain('The checks failed'); + expect(timedOut.body).toContain('base not checked'); + // Without the timings it still says timeout, not failure. + const bare = report('timeout', 'unknown'); + expect(bare.body).toContain('**The checks timed out**: they were stopped at their time budget'); + expect(bare.body).not.toContain('The checks failed'); + // Not run: the flow ran out of time before the checks (relaycast-cloud#216). + const notRun = report('skipped', ''); + expect(notRun.body).toContain("ran out of time before it could run this repository's checks"); + expect(notRun.body).not.toContain('timed out'); + expect(notRun.body).not.toContain('The checks failed'); + }); + it('keeps the base commit\'s output when its verdict is unknown, such as a base check that ran out of time', () => { const unknown = report('fail', 'unknown'); expect(unknown.body).toContain('could not be checked for comparison'); @@ -731,6 +961,20 @@ describe('FLOW_CHECK_REPORT_COMMAND', () => { expect(read(root, '.relayflow/pr-body.md')).not.toContain('implementer was stopped'); }); + it('says the checks could not be run, not that they failed, when nothing could bound them', () => { + const { body, code } = report('unrunnable', '', { '.relayflow/check.log': 'relayflow: this machine has no timeout, gtimeout or perl, so there is no way to stop the checks at a time limit and they were not run.\n' }); + expect(code).toBe(0); + expect(body).toContain("**Relayflow could not run this repository's checks**"); + expect(body).toContain('no `timeout`, `gtimeout` or `perl`'); + expect(body).toContain('This is not a test failure'); + expect(body).toContain('Install coreutils'); + expect(body).not.toContain('The checks failed'); + expect(body).not.toContain('timed out'); + // What ran is still shown; there is no suite output to show. + expect(body).toContain('npm test'); + expect(body).not.toContain('Output on this branch'); + }); + it('says plainly when nothing could be checked', () => { const { body } = report('none', '', { [FLOW_CHECK_SCRIPT]: '' }); expect(body).toContain('no checks ran'); diff --git a/web/public/flows/software-garden/manifest.json b/web/public/flows/software-garden/manifest.json index b43b6346..27be09c1 100644 --- a/web/public/flows/software-garden/manifest.json +++ b/web/public/flows/software-garden/manifest.json @@ -1,11 +1,11 @@ { "name": "Software Garden", "catalogId": "software-factory", - "version": 1, - "sha256": "b89a36e2cf9054036a7fc53fba20011a42ef0275ffbaaa907b080738ab46ad27", - "bytes": 64355, - "path": "web/public/flows/software-garden/v1.flow.ts", - "url": "https://agentrelay.com/flows/software-garden/v1.flow.ts", + "version": 2, + "sha256": "516ccd18d39e7f03df20442963fe339cfc64fcd09fa3659e53d85cf6a31d60fb", + "bytes": 87545, + "path": "web/public/flows/software-garden/v2.flow.ts", + "url": "https://agentrelay.com/flows/software-garden/v2.flow.ts", "target": "cloud", "draft": { "version": 4, @@ -25,6 +25,10 @@ { "version": 1, "sha256": "b89a36e2cf9054036a7fc53fba20011a42ef0275ffbaaa907b080738ab46ad27" + }, + { + "version": 2, + "sha256": "516ccd18d39e7f03df20442963fe339cfc64fcd09fa3659e53d85cf6a31d60fb" } ] } diff --git a/web/public/flows/software-garden/v2.flow.ts b/web/public/flows/software-garden/v2.flow.ts new file mode 100644 index 00000000..687cddd0 --- /dev/null +++ b/web/public/flows/software-garden/v2.flow.ts @@ -0,0 +1,376 @@ +import { flow } from "@relayflows/surface"; + +// The ticket this flow works on. +export type Issue = { + source: string; + title: string; + body: string; + labels: string[]; + repository?: string; + // Normalized by Cloud for every connected ticket provider. A local input + // may leave either field blank when that provider has no such metadata. + identifier?: string; + url?: string; +}; + +// Your coding agent, ready to work. +const builder = "claude"; + +type Input = { issue: Issue; approver: string }; + +// Run in a connected repository, on a new branch. +export default flow("software-factory", + { budget: { wallclock: "2h" } }, async (f, input) => { + const issue = input.issue; + // Cloud starts this flow only for tickets that already match the sources + // and filters you chose, so just check the ticket arrived intact. + if (!issue?.title?.trim()) { + // Parked, not canceled: the runtime refuses done("canceled") by design, as + // a kernel fact a flow body cannot declare (AgentWorkforce/flows#436). The + // reason this wants is a declination — AgentWorkforce/flows#438, unshipped. + console.error("Stopped: no ticket arrived with this run, so there was nothing to work on."); + return f.done("needs_human"); + } + + const normalizedTitle = issue.title.trim().replace(/\s+/g, " "); + // Bound and measure by Unicode code points so neither truncation nor the + // final shell validation can split or byte-count a multibyte character. + const changeTitle = Array.from(normalizedTitle).slice(0, 240).join("").trim(); + const changeTitleLength = Array.from(changeTitle).length; + const placeholderTitle = ["software factory change", "replace with your ticket title"] + .includes(changeTitle.toLowerCase()); + const issueSource = issue.source.trim().toLowerCase(); + const issueIdentifier = issue.identifier?.trim() ?? ""; + const issueUrl = issue.url?.trim() ?? ""; + if (!changeTitle || placeholderTitle) { + console.error("Stopped: the pull-request title is empty or still a placeholder. No branch was pushed and no pull request was opened."); + return f.done("needs_human"); + } + if (issueSource === "github" && !/^#[1-9]\d*$/.test(issueIdentifier)) { + console.error("Stopped: a GitHub ticket must carry its normalized identifier in # form. No branch was pushed and no pull request was opened."); + return f.done("needs_human"); + } + // GitHub receives its exact closing keyword. Other providers get a stable + // native reference when one is available; Markdown invents no identifier. + const changeReference = issueSource === "github" + ? "Fixes " + issueIdentifier + : issueSource === "gitlab" && /^#[1-9]\d*$/.test(issueIdentifier) + ? "Closes " + issueIdentifier + : issueUrl + ? "Ticket: " + issueUrl + : issueIdentifier + ? "Ticket: " + issueIdentifier + : ""; + const task = issue.title + "\n" + issue.body + "\n" + + "Follow existing patterns. Keep changes focused and add regression tests." + "\n" + + "Changes under .github/workflows/ may not be pushable by this run's token, so do not edit workflow files unless the task requires it; if you do, keep those edits in separate commits."; + // Files the agents write for each other (summary.md, reviews, and + // .relayflow/) are never part of the change; keep them out of every commit. + await f.run("x=$(git rev-parse --git-path info/exclude 2>/dev/null); if [ -n \"$x\" ]; then mkdir -p \"$(dirname \"$x\")\" 2>/dev/null; if ! grep -qxF '# relayflow working files' \"$x\" 2>/dev/null; then printf '%s\\n' '' '# relayflow working files' '/summary.md' '/plan.md' '/reviewed-plan.md' '/review.md' '/review.clean' '/review-blocked.md' '/comparison.md' '/prototype-notes.md' '/.relayflow/' '/START-HERE.txt' '/flow-input.json' '/relay-preflight.mjs' '/software-factory.flow.mts' '/.relayflowd/' >> \"$x\"; fi; echo \"relayflow: working files are excluded from commits.\" >&2; else echo \"relayflow: not a git repository, so nothing was excluded.\" >&2; fi; echo done"); + // The budget in the header is spent step by step, and once it is gone no + // step runs, not even the push that keeps the work. So each long step below + // gets only what is left once the steps behind it, and publishing, still + // fit. The clock is read through a step so a resumed run replays it. + const clock = async () => Number((await f.run("date +%s")).trim()); + const startedAt = await clock(); + // Parallel steps are charged once each, so they count more than the clock. + let parallelMinutes = 0; + const minutesLeft = async () => 112 - (await clock() - startedAt) / 60 - parallelMinutes; + // The most a long step may take, in whole minutes, when `after` minutes + // must still fit behind it: at most `cap`, and below 1 once nothing is left. + const allowance = async (cap: number, after: number) => Math.min(cap, Math.floor(await minutesLeft() - after)); + // Publishing, and the steps between a long step and it. + const publishing = 26; + // What the steps up to the implementer leave: on Cloud, everything after + // minute 34, so the build takes no longer than it did in 60 minutes. + const building = 78; + // A long agent step may be stopped at its time limit. Its step then resolves + // with completionReason "timeout" instead of throwing, and what the agent + // committed stays. That is never success: each caller says what it means. + const timedOut = (result: unknown) => (result as { completionReason?: string } | undefined)?.completionReason === "timeout"; + + // Work out how this repository checks itself, before any code changes, so + // the change and the commit it started from are checked the same way. + // To use your own command instead, set checkCommand, for example + // "npm run build && npm test" or "make ci". + const checkCommand = ""; + const shellQuote = (value: string) => "'" + value.replace(/'/g, "'\\''") + "'"; + const checkScript = ".relayflow/check.sh"; + if (checkCommand.trim()) { + await f.run("mkdir -p .relayflow && printf '%s\\n' 'set -e' " + shellQuote(checkCommand) + " > " + checkScript); + } else if ((await f.run("test -s " + checkScript + " && echo yes || echo no")).trim() !== "yes") { + // No command of yours and none committed to the repository: read how its + // CI tests it. Anything this cannot settle falls back to a default below. + const discovery = await f.agent("check-discovery", { + cli: builder, + model: "claude-sonnet-5", + task: task + "\n" + "This is a setup step: do not start on the ticket. Work out how this repository checks itself on a fresh machine, the way its CI does. Read the CI configuration (.github/workflows, .gitlab-ci.yml, .circleci and similar), any Makefile, justfile or Taskfile, and AGENTS.md, CLAUDE.md, CONTRIBUTING and README. Write .relayflow/check.sh: a POSIX sh script starting with set -e that installs dependencies, runs whatever CI runs before its tests (builds, code generation), then runs the tests. Leave out steps that need secrets, deployments or services this machine does not have, with a comment saying why. Do not run the full test suite, do not change any other file, and do not commit. If the repository has no tests, do not create the file.", + timeout: "15m", + }); + if (timedOut(discovery)) { + // A script it was stopped while writing is not a recipe: discard it, + // so the ecosystem default below is used instead. + console.error("check-discovery was stopped at its 15m limit. Any check script it left is discarded, and the ecosystem default is used."); + await f.run("rm -f " + checkScript); + } + } + const resolveChecks = "mkdir -p .relayflow; has_test_target() { for f in GNUmakefile makefile Makefile; do if [ -f \"$f\" ] && grep -Eq '^test[[:space:]]*:' \"$f\"; then return 0; fi; done; return 1; }; eco=; cmd=; if [ -s .relayflow/check.sh ]; then eco=script; elif has_test_target; then eco=make; cmd='make test'; elif [ -f justfile ] && grep -Eq '^test([[:space:]]|:)' justfile; then eco=just; cmd='just test'; elif [ -f package.json ] && node -e 'const f=require(\"fs\");let p;try{p=JSON.parse(f.readFileSync(\"package.json\",\"utf8\"))}catch{process.exit(1)}const t=(p.scripts||{}).test;process.exit(t&&String(t).trim()?0:1)' >/dev/null 2>&1; then eco=node; elif [ -f Cargo.toml ]; then eco=cargo; cmd='cargo test'; elif [ -f go.mod ]; then eco=go; cmd='go test ./...'; elif [ -f pyproject.toml ] || [ -f setup.py ] || [ -f setup.cfg ] || [ -f requirements.txt ] || [ -f pytest.ini ] || [ -f tox.ini ]; then eco=python; if [ -f uv.lock ]; then cmd='uv run pytest'; elif [ -f poetry.lock ]; then cmd='poetry install --no-interaction && poetry run pytest'; elif [ -f requirements.txt ]; then cmd='python3 -m pip install -r requirements.txt && python3 -m pytest'; else cmd='python3 -m pytest'; fi; elif [ -f Gemfile ]; then eco=ruby; if [ -d spec ]; then cmd='bundle install && bundle exec rspec'; else cmd='bundle install && bundle exec rake test'; fi; elif [ -f pom.xml ]; then eco=maven; if [ -x mvnw ]; then cmd='./mvnw -B test'; else cmd='mvn -B test'; fi; elif [ -f build.gradle ] || [ -f build.gradle.kts ]; then eco=gradle; if [ -x gradlew ]; then cmd='./gradlew test'; else cmd='gradle test'; fi; elif [ -n \"$(find . -maxdepth 1 \\( -name '*.sln' -o -name '*.csproj' -o -name '*.fsproj' \\) -print 2>/dev/null | head -n 1)\" ]; then eco=dotnet; cmd='dotnet test'; elif [ -f mix.exs ]; then eco=elixir; cmd='mix deps.get && mix test'; fi; if [ \"$eco\" = script ]; then echo \"relayflow: checking with the repository's own .relayflow/check.sh.\" >&2; echo script; elif [ -z \"$eco\" ]; then echo \"relayflow: found no way to run this repository's tests: no check command, no test target and no recognised project file.\" >&2; echo none; elif [ \"$eco\" = node ]; then printf '%s\\n' '# Written by relayflow from the files in this repository. Edit it to match how CI runs the tests.' 'set -e' 'if [ -f pnpm-lock.yaml ]; then pm=pnpm; elif [ -f yarn.lock ]; then pm=yarn; elif [ -f bun.lock ] || [ -f bun.lockb ]; then pm=bun; else pm=npm; fi' 'if [ \"$pm\" = pnpm ] || [ \"$pm\" = yarn ]; then if ! command -v \"$pm\" >/dev/null 2>&1; then mkdir -p \"$HOME/.local/bin\" && { corepack enable --install-directory \"$HOME/.local/bin\" \"$pm\" >/dev/null 2>&1 || true; } && PATH=\"$HOME/.local/bin:$PATH\" && export PATH; fi; COREPACK_ENABLE_DOWNLOAD_PROMPT=0 && export COREPACK_ENABLE_DOWNLOAD_PROMPT; fi' 'if ! command -v \"$pm\" >/dev/null 2>&1; then echo \"relayflow: this repository'\\''s lockfile requires $pm, which is not installed and could not be provisioned.\" >&2; exit 1; fi' 'echo \"relayflow: installing dependencies and running tests with $pm\"' 'if [ \"$pm\" = pnpm ]; then pnpm install --frozen-lockfile && pnpm test; elif [ \"$pm\" = yarn ]; then { if [ -f .yarnrc.yml ]; then yarn install --immutable; else yarn install --frozen-lockfile; fi; } && yarn test; elif [ \"$pm\" = bun ]; then bun install --frozen-lockfile && bun run test; else { if [ -f package-lock.json ]; then npm ci; else npm install; fi; } && npm test; fi' > .relayflow/check.sh; echo \"relayflow: wrote the Node default to .relayflow/check.sh.\" >&2; echo default; else printf '%s\\n' '# Written by relayflow from the files in this repository. Edit it to match how CI runs the tests.' 'set -e' \"$cmd\" > .relayflow/check.sh; echo \"relayflow: wrote the $eco default ($cmd) to .relayflow/check.sh.\" >&2; echo default; fi"; + const checkPlan = (await f.run(resolveChecks)).trim(); + // Removes build outputs the run no longer needs before a check, so a second + // build cannot fill the sandbox's disk. + const freeDisk = "relayflow_build_dirs() { ( cd \"$1\" 2>/dev/null && find . -maxdepth 3 \\( -name .git -o -name node_modules \\) -prune -o -type d -exec test -f {}/CACHEDIR.TAG \\; -print -prune 2>/dev/null | while IFS= read -r d; do if git check-ignore -q -- \"$d/\" 2>/dev/null; then printf '%s\\n' \"${d#./}\"; fi; done ); }; relayflow_drop_builds() { relayflow_build_dirs \"$1\" | while IFS= read -r d; do rm -rf -- \"$1/$d\" && echo \"relayflow: removed $1/$d\" >&2; done; }; for t in ${dirs:-}; do if [ -d \"$t\" ]; then relayflow_drop_builds \"$t\"; fi; done; git worktree prune >/dev/null 2>&1; min=\"${RELAYFLOW_MIN_FREE_MB:-4096}\"; avail=$(df -Pk . 2>/dev/null | awk 'NR == 2 { print $4 }'); if printf '%s\\n' \"$avail\" | grep -Eq '^[0-9]+$' && [ \"$avail\" -lt $((min * 1024)) ]; then echo \"relayflow: low on disk ($((avail / 1024)) MB free), so this tree's build directories are removed; the next check rebuilds.\" >&2; relayflow_drop_builds \"$PWD\"; fi; echo done"; + + // Build and test the change, then open a pull request. + // Where this branch started, so the publish step below can tell whether the + // agents actually committed anything. + const baseCommit = (await f.run("git rev-parse HEAD")).trim(); + // The implementer may use what is left once what follows the build fits. + // Stopped there, its committed work is still checked, and published as a + // draft that says so. + const implementerLimit = Math.max(1, await allowance(60, building)); + const implementation = await f.agent("implementer", { + cli: builder, + model: "claude-sonnet-5", + task: task + "\n" + "Start by reading the ticket and the code it touches, and plan the change before you write it. Implement on the current branch. Add regression tests. Commit changes. Write a PR summary to summary.md.", + timeout: implementerLimit + "m", + }); + const implementerTimedOut = timedOut(implementation); + if (implementerTimedOut) console.error("implementer was stopped at its " + implementerLimit + "m limit. Its committed work is kept, and the pull request opens as a draft."); + // A repository with no way to test itself may have gained one in this + // change (a first package.json with a test script). Resolving only before + // the change reported "no checks ran" on a pull request that added tests + // (cloud-e2e-sandbox#31), so a "none" is looked at again. + if (checkPlan === "none") await f.run(resolveChecks); + + // Run the checks. A failure is not the end of the run: the tests are + // how this flow learns what is wrong, so the repair agent reads the output + // and fixes what it can, at most twice, and whatever still fails is compared against + // the commit this branch started from when there is time. Every check step + // prints one word and exits 0; the full output stays in .relayflow/ for the + // report. Each branch check gets up to checkTotal minutes, what is left once + // publishing and the review's floor fit, and stops there. + const verdict = (value: string) => ["pass", "fail", "timeout", "none", "unrunnable"].includes(value.trim()) ? value.trim() : "fail"; + const verdictOf = (value: string) => /^[a-z]*$/.test(value) ? value : "unknown"; + const broken = (value: string) => value === "fail" || value === "timeout"; + const runChecks = "root=\"$PWD\"; script=\"${check_script:-$root/.relayflow/check.sh}\"; check_dir=\"${check_dir:-$root}\"; check_out=\"${check_out:-$root/.relayflow/check.log}\"; mkdir -p \"$(dirname \"$check_out\")\"; rm -f \"$check_out.exit\" \"$check_out.elapsed\" \"$check_out.limit\"; if [ ! -s \"$script\" ]; then echo \"relayflow: there is no check script, so no checks ran.\" > \"$check_out\"; echo \"relayflow: there is no check script, so no checks ran.\" >&2; echo none; else limit=\"${RELAYFLOW_CHECK_TIMEOUT:-840}\"; limiter=; if command -v timeout >/dev/null 2>&1; then limiter=timeout; elif command -v gtimeout >/dev/null 2>&1; then limiter=gtimeout; elif command -v perl >/dev/null 2>&1; then limiter=perl; fi; relayflow_limiter='my $l = shift; my $p = fork; defined $p or exit 127; if (!$p) { setpgrp(0, 0); exec @ARGV or exit 127 } $SIG{ALRM} = sub { kill '\\''TERM'\\'', -$p; sleep 2; kill '\\''KILL'\\'', -$p; exit 124 }; alarm $l; waitpid($p, 0); exit($? & 127 ? 128 + ($? & 127) : $? >> 8)'; grouper=; if command -v setsid >/dev/null 2>&1; then grouper=setsid; elif command -v perl >/dev/null 2>&1; then grouper=perl; fi; relayflow_inline=; relayflow_check_done() { printf '%s\\n' \"$1\" > \"$check_out.exit\"; printf '%s\\n' \"$2\" > \"$check_out.elapsed\"; printf '%s\\n' \"$limit\" > \"$check_out.limit\"; tail -n 40 \"$check_out\" >&2; if [ \"$1\" -eq 0 ]; then echo \"relayflow: the checks passed.\" >&2; echo pass; elif [ \"$1\" -eq 124 ] && { { [ \"$relayflow_inline\" = yes ] && [ -n \"$limiter\" ]; } || [ \"${3:-}\" = stopped ]; }; then echo \"relayflow: the checks did not finish within ${limit}s (they ran $2s).\" >&2; echo timeout; else echo \"relayflow: the checks failed with exit $1; the full output is in $check_out.\" >&2; echo fail; fi; }; if [ -z \"$limiter\" ]; then relayflow_unrunnable=\"relayflow: this machine has no timeout, gtimeout or perl, so there is no way to stop the checks at a time limit and they were not run. Install coreutils (timeout) or perl, then run again.\"; printf '%s\\n' \"$relayflow_unrunnable\" > \"$check_out\"; echo \"$relayflow_unrunnable\" >&2; echo unrunnable; elif [ -z \"${RELAYFLOW_CHECK_WAIT:-}\" ] || [ -z \"$grouper\" ]; then relayflow_inline=yes; if [ -n \"${RELAYFLOW_CHECK_WAIT:-}\" ]; then if [ \"$RELAYFLOW_CHECK_WAIT\" -lt \"$limit\" ]; then limit=\"$RELAYFLOW_CHECK_WAIT\"; fi; echo \"relayflow: neither setsid nor perl is installed, so the checks cannot be detached; they run in this step under $limiter, limited to ${limit}s.\" >&2; fi; echo \"relayflow: running $script in $check_dir\" >&2; since=$(date +%s); ( cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; case \"$limiter\" in (timeout|gtimeout) \"$limiter\" \"$limit\" sh \"$script\" ;; (perl) perl -e \"$relayflow_limiter\" \"$limit\" sh \"$script\" ;; (*) sh \"$script\" ;; esac > \"$check_out\" 2>&1 < /dev/null; printf \"%s\\n\" \"$?\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\" ); relayflow_check_done \"$(cat \"$check_out.status\" 2>/dev/null || echo 125)\" \"$(( $(date +%s) - since ))\"; rm -f \"$check_out.status\"; else id=\"${RELAYFLOW_CHECK_ID:-1}\"; relayflow_group() { cat \"$check_out.group\" 2>/dev/null || cat \"$check_out.pid\" 2>/dev/null; }; relayflow_alive() { g=$(relayflow_group); [ -n \"$g\" ] && { kill -0 \"-$g\" 2>/dev/null || kill -0 \"$g\" 2>/dev/null; }; }; relayflow_stop() { g=$(relayflow_group); [ -n \"$g\" ] || return 0; kill -TERM \"-$g\" 2>/dev/null || kill -TERM \"$g\" 2>/dev/null; i=0; while relayflow_alive && [ \"$i\" -lt 5 ]; do sleep 1; i=$((i + 1)); done; kill -KILL \"-$g\" 2>/dev/null || kill -KILL \"$g\" 2>/dev/null; return 0; }; relayflow_status_done() { if [ -f \"$check_out.stopped\" ]; then relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\" stopped; else relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\"; fi; }; if [ \"$(cat \"$check_out.id\" 2>/dev/null)\" != \"$id\" ]; then if [ ! -f \"$check_out.status\" ] && relayflow_alive; then if ! command -v ps >/dev/null 2>&1; then relayflow_stop; echo \"relayflow: stopped what an earlier attempt left running (no ps here to identify it).\" >&2; elif ps -o args= -p \"$(relayflow_group)\" 2>/dev/null | grep -qF 'check_out.group'; then relayflow_stop; echo \"relayflow: stopped a check an earlier attempt left running.\" >&2; fi; fi; rm -f \"$check_out.status\" \"$check_out.status.tmp\" \"$check_out.stopped\" \"$check_out.group\" \"$check_out.pid\"; : > \"$check_out\"; date +%s > \"$check_out.since\"; printf '%s\\n' \"$id\" > \"$check_out.id\"; export check_dir check_out script limit; if [ \"$grouper\" = setsid ]; then setsid sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & else perl -e '$SIG{HUP} = \"IGNORE\"; setpgrp(0, 0); exec @ARGV or exit 127' sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & fi; echo \"$!\" > \"$check_out.pid\"; echo \"relayflow: started $script in $check_dir; it may run ${limit}s in all, and each step waits up to ${RELAYFLOW_CHECK_WAIT}s for it.\" >&2; fi; since=$(cat \"$check_out.since\" 2>/dev/null || date +%s); waited=0; while [ ! -f \"$check_out.status\" ] && [ \"$waited\" -lt \"$RELAYFLOW_CHECK_WAIT\" ] && relayflow_alive && [ $(( $(date +%s) - since )) -lt \"$limit\" ]; do sleep 1; waited=$((waited + 1)); done; elapsed=$(( $(date +%s) - since )); if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; elif relayflow_alive && [ \"$elapsed\" -lt \"$limit\" ]; then echo \"relayflow: the checks are still running (${elapsed}s of ${limit}s).\" >&2; echo running; elif relayflow_alive; then : > \"$check_out.stopped\"; printf '%s\\n' 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; relayflow_stop; echo \"relayflow: stopped the checks after ${elapsed}s, at their ${limit}s.\" >> \"$check_out\"; relayflow_check_done 124 \"$elapsed\" stopped; else sleep 1; if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; else echo \"relayflow: the checks stopped without reporting an exit status.\" >> \"$check_out\"; relayflow_check_done 125 \"$elapsed\"; fi; fi; fi; fi"; + // How long the last check took. A re-check runs in the same tree and reuses + // its build, and the base commit's check runs the same recipe, so each is + // budgeted like it. + let checkTook = 0; + // The f.run lease is longer than the check's own limit, so the check stops + // itself and reports "timeout" first. + const timedCheck = async (prefix: string, command: string, minutes: number, lease = minutes + 1) => { + const from = await clock(); + const result = (await f.run(prefix + "RELAYFLOW_CHECK_TIMEOUT=" + minutes * 60 + "; " + command, { timeout: lease + "m" })).trim(); + checkTook = (await clock() - from) / 60; + return result; + }; + // The most a branch check may run in all. To give a slow suite more, raise + // it; the run's time budget still decides what each check gets. + const checkTotal = 30; + // A branch check may need more than one f.run lease (flows#626), so it runs + // detached and each step waits at most 14 minutes for it, until it + // finishes or its total runs out (reported as "timeout"). Each check has its + // own ID, so a resumed run waits for the check it started. + let checkRuns = 0; + const spannedCheck = async (minutes: number) => { + const from = await clock(); + const id = startedAt + "-" + (++checkRuns); + let result = "running"; + for (let waited = 0; result === "running" && waited <= minutes; ) { + const wait = Math.min(14, minutes + 1 - waited); + waited += wait; + result = (await f.run("RELAYFLOW_CHECK_ID=" + id + "; RELAYFLOW_CHECK_TIMEOUT=" + minutes * 60 + "; RELAYFLOW_CHECK_WAIT=" + wait * 60 + "; " + runChecks, { timeout: wait + 1 + "m" })).trim(); + } + checkTook = (await clock() - from) / 60; + return result === "running" ? "timeout" : result; + }; + // Behind each check: the two clock reads around it and each wait's minute + // of lease beyond it, then publishing, and the review's floor where there + // is one (its whole allowance locally, where nothing stops it sooner). + const checkSlack = 4; + const reviewing = 10.5; + let check = "skipped"; + const checkLimit = await allowance(checkTotal, checkSlack + publishing + reviewing); + if (checkLimit < 5) { + console.error("Skipped the checks: too little of the flow's time budget is left to run them and still publish. The pull request opens as a draft that says no checks ran."); + } else { + check = verdict(await spannedCheck(checkLimit)); + // No way to bound the suite, so it was never started: not a verdict on + // this change, and nothing to repair or compare. The draft says so. + if (check === "unrunnable") console.error("The checks could not be run: this machine has no timeout, gtimeout or perl to stop them at a time limit, so starting them could have ended the run with no verdict. The pull request opens as a draft that says so."); + } + // A second repair round runs only for checks that still fail after the + // first, and leaves the review, where there is one, its floor: it is + // recovery, never a longer plan. + // The base-commit check that may follow, as compareWithBase budgets it: + // nothing when it would be skipped anyway. + const baseReserve = () => checkPlan === "none" || Math.ceil(checkTook) > 13 ? 0 : Math.max(5, Math.ceil(checkTook)) + 5; + for (let round = 1; round <= 2 && broken(check); round++) { + // Each round: the repair, freeing disk, the re-check, publishing and the + // review must fit in what is left; a second round also keeps the base + // check's time, and its re-check is held to what was kept for it. + const recheck = Math.min(checkTotal, Math.ceil(checkTook) + 1); + const baseHold = round > 1 ? baseReserve() : 0; + const repairLimit = await allowance(20, recheck + 1 + checkSlack + publishing + reviewing + baseHold); + if (repairLimit < 5) { + console.error(round === 1 + ? "Skipped the repair: too little of the flow's time budget is left to repair and still publish. The checks stand as they are." + : "Skipped the second repair: too little of the flow's time budget is left to repair again and still publish and review. The checks stand as they are."); + break; + } + const repair = await f.agent("check-repair" + (round === 1 ? "" : "-" + round), { + cli: builder, + model: "claude-sonnet-5", + task: task + "\n" + "The repository's checks failed on this branch. The command that ran is .relayflow/check.sh and its full output is in .relayflow/check.log. For each failure, work out whether it comes from missing setup (a build, code generation or install step the tests expect, often named in the error) or from a bug in the change on this branch. Fix missing setup by adding the step to .relayflow/check.sh the way the repository's CI does it; do not commit that file. Fix bugs in the change and commit the fix. Never skip, delete or weaken a test, and never change a test only to make it pass. If a failure is outside your control, such as a tool that is not installed, no network, or missing credentials, leave it and write what you found to .relayflow/repair-notes.md.", + timeout: repairLimit + "m", + }); + await f.run(freeDisk); + const recheckLimit = await allowance(checkTotal, checkSlack + publishing + reviewing + baseHold); + if (recheckLimit >= 1) check = verdict(await spannedCheck(recheckLimit)); + else console.error("No time was left to check the repair again; the checks stand as they were before it."); + if (timedOut(repair)) { + // A failed attempt: what it committed was checked again above, and + // another repair of the same failures would most likely run out too, + // so whatever still fails takes the draft-and-report path below. + console.error("check-repair was stopped at its " + repairLimit + "m limit, so it counts as a failed repair attempt. Its committed work is kept and was checked again (" + check + "); no further repair is tried."); + break; + } + } + // Only a failure pays for the comparison: pass, fail, timeout or unknown. + // Checks this change introduced (nothing to run before it) have no base to + // compare with: the base lacks the files they need and would always fail, + // which read as "pre-existing". Their failure is this change's own. With too + // little time to check the base commit as long as the branch took and still + // publish, the report says the base was not checked, and the draft opens. + const compareWithBase = async () => { + if (!broken(check)) return ""; + if (checkPlan === "none") return "new"; + // Checking out the base commit and returning to the branch share the + // check's lease, so it gets two minutes beyond the limit, within f.run's + // 15. It never gets less time than the branch's check took: a branch check + // that ran to its own limit cannot be compared at all. + const baseLimit = await allowance(13, 4 + publishing + reviewing); + if (baseLimit < Math.max(5, Math.ceil(checkTook))) return "skipped"; + await f.run(freeDisk); + const result = await timedCheck("base=" + baseCommit + "; ", "root=\"$PWD\"; tmp=; relayflow_build_dirs() { ( cd \"$1\" 2>/dev/null && find . -maxdepth 3 \\( -name .git -o -name node_modules \\) -prune -o -type d -exec test -f {}/CACHEDIR.TAG \\; -print -prune 2>/dev/null | while IFS= read -r d; do if git check-ignore -q -- \"$d/\" 2>/dev/null; then printf '%s\\n' \"${d#./}\"; fi; done ); }; if [ -z \"$base\" ] || ! git rev-parse --verify --quiet \"$base^{commit}\" >/dev/null 2>&1; then echo \"relayflow: could not check out the base commit to compare against.\" >&2; echo unknown; elif git diff --quiet HEAD -- >/dev/null 2>&1 && git diff --cached --quiet >/dev/null 2>&1 && head=$(git rev-parse HEAD) && orig=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) && recipe=$(mktemp \"${TMPDIR:-/tmp}/relayflow-recipe.XXXXXX\") && { [ ! -f \"$root/.relayflow/check.sh\" ] || cp \"$root/.relayflow/check.sh\" \"$recipe\"; } && git checkout -q --detach \"$base\" >/dev/null 2>&1; then check_dir=\"$root\"; check_out=\"$root/.relayflow/base-check.log\"; check_script=\"$recipe\"; token=$( root=\"$PWD\"; script=\"${check_script:-$root/.relayflow/check.sh}\"; check_dir=\"${check_dir:-$root}\"; check_out=\"${check_out:-$root/.relayflow/check.log}\"; mkdir -p \"$(dirname \"$check_out\")\"; rm -f \"$check_out.exit\" \"$check_out.elapsed\" \"$check_out.limit\"; if [ ! -s \"$script\" ]; then echo \"relayflow: there is no check script, so no checks ran.\" > \"$check_out\"; echo \"relayflow: there is no check script, so no checks ran.\" >&2; echo none; else limit=\"${RELAYFLOW_CHECK_TIMEOUT:-840}\"; limiter=; if command -v timeout >/dev/null 2>&1; then limiter=timeout; elif command -v gtimeout >/dev/null 2>&1; then limiter=gtimeout; elif command -v perl >/dev/null 2>&1; then limiter=perl; fi; relayflow_limiter='my $l = shift; my $p = fork; defined $p or exit 127; if (!$p) { setpgrp(0, 0); exec @ARGV or exit 127 } $SIG{ALRM} = sub { kill '\\''TERM'\\'', -$p; sleep 2; kill '\\''KILL'\\'', -$p; exit 124 }; alarm $l; waitpid($p, 0); exit($? & 127 ? 128 + ($? & 127) : $? >> 8)'; grouper=; if command -v setsid >/dev/null 2>&1; then grouper=setsid; elif command -v perl >/dev/null 2>&1; then grouper=perl; fi; relayflow_inline=; relayflow_check_done() { printf '%s\\n' \"$1\" > \"$check_out.exit\"; printf '%s\\n' \"$2\" > \"$check_out.elapsed\"; printf '%s\\n' \"$limit\" > \"$check_out.limit\"; tail -n 40 \"$check_out\" >&2; if [ \"$1\" -eq 0 ]; then echo \"relayflow: the checks passed.\" >&2; echo pass; elif [ \"$1\" -eq 124 ] && { { [ \"$relayflow_inline\" = yes ] && [ -n \"$limiter\" ]; } || [ \"${3:-}\" = stopped ]; }; then echo \"relayflow: the checks did not finish within ${limit}s (they ran $2s).\" >&2; echo timeout; else echo \"relayflow: the checks failed with exit $1; the full output is in $check_out.\" >&2; echo fail; fi; }; if [ -z \"$limiter\" ]; then relayflow_unrunnable=\"relayflow: this machine has no timeout, gtimeout or perl, so there is no way to stop the checks at a time limit and they were not run. Install coreutils (timeout) or perl, then run again.\"; printf '%s\\n' \"$relayflow_unrunnable\" > \"$check_out\"; echo \"$relayflow_unrunnable\" >&2; echo unrunnable; elif [ -z \"${RELAYFLOW_CHECK_WAIT:-}\" ] || [ -z \"$grouper\" ]; then relayflow_inline=yes; if [ -n \"${RELAYFLOW_CHECK_WAIT:-}\" ]; then if [ \"$RELAYFLOW_CHECK_WAIT\" -lt \"$limit\" ]; then limit=\"$RELAYFLOW_CHECK_WAIT\"; fi; echo \"relayflow: neither setsid nor perl is installed, so the checks cannot be detached; they run in this step under $limiter, limited to ${limit}s.\" >&2; fi; echo \"relayflow: running $script in $check_dir\" >&2; since=$(date +%s); ( cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; case \"$limiter\" in (timeout|gtimeout) \"$limiter\" \"$limit\" sh \"$script\" ;; (perl) perl -e \"$relayflow_limiter\" \"$limit\" sh \"$script\" ;; (*) sh \"$script\" ;; esac > \"$check_out\" 2>&1 < /dev/null; printf \"%s\\n\" \"$?\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\" ); relayflow_check_done \"$(cat \"$check_out.status\" 2>/dev/null || echo 125)\" \"$(( $(date +%s) - since ))\"; rm -f \"$check_out.status\"; else id=\"${RELAYFLOW_CHECK_ID:-1}\"; relayflow_group() { cat \"$check_out.group\" 2>/dev/null || cat \"$check_out.pid\" 2>/dev/null; }; relayflow_alive() { g=$(relayflow_group); [ -n \"$g\" ] && { kill -0 \"-$g\" 2>/dev/null || kill -0 \"$g\" 2>/dev/null; }; }; relayflow_stop() { g=$(relayflow_group); [ -n \"$g\" ] || return 0; kill -TERM \"-$g\" 2>/dev/null || kill -TERM \"$g\" 2>/dev/null; i=0; while relayflow_alive && [ \"$i\" -lt 5 ]; do sleep 1; i=$((i + 1)); done; kill -KILL \"-$g\" 2>/dev/null || kill -KILL \"$g\" 2>/dev/null; return 0; }; relayflow_status_done() { if [ -f \"$check_out.stopped\" ]; then relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\" stopped; else relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\"; fi; }; if [ \"$(cat \"$check_out.id\" 2>/dev/null)\" != \"$id\" ]; then if [ ! -f \"$check_out.status\" ] && relayflow_alive; then if ! command -v ps >/dev/null 2>&1; then relayflow_stop; echo \"relayflow: stopped what an earlier attempt left running (no ps here to identify it).\" >&2; elif ps -o args= -p \"$(relayflow_group)\" 2>/dev/null | grep -qF 'check_out.group'; then relayflow_stop; echo \"relayflow: stopped a check an earlier attempt left running.\" >&2; fi; fi; rm -f \"$check_out.status\" \"$check_out.status.tmp\" \"$check_out.stopped\" \"$check_out.group\" \"$check_out.pid\"; : > \"$check_out\"; date +%s > \"$check_out.since\"; printf '%s\\n' \"$id\" > \"$check_out.id\"; export check_dir check_out script limit; if [ \"$grouper\" = setsid ]; then setsid sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & else perl -e '$SIG{HUP} = \"IGNORE\"; setpgrp(0, 0); exec @ARGV or exit 127' sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & fi; echo \"$!\" > \"$check_out.pid\"; echo \"relayflow: started $script in $check_dir; it may run ${limit}s in all, and each step waits up to ${RELAYFLOW_CHECK_WAIT}s for it.\" >&2; fi; since=$(cat \"$check_out.since\" 2>/dev/null || date +%s); waited=0; while [ ! -f \"$check_out.status\" ] && [ \"$waited\" -lt \"$RELAYFLOW_CHECK_WAIT\" ] && relayflow_alive && [ $(( $(date +%s) - since )) -lt \"$limit\" ]; do sleep 1; waited=$((waited + 1)); done; elapsed=$(( $(date +%s) - since )); if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; elif relayflow_alive && [ \"$elapsed\" -lt \"$limit\" ]; then echo \"relayflow: the checks are still running (${elapsed}s of ${limit}s).\" >&2; echo running; elif relayflow_alive; then : > \"$check_out.stopped\"; printf '%s\\n' 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; relayflow_stop; echo \"relayflow: stopped the checks after ${elapsed}s, at their ${limit}s.\" >> \"$check_out\"; relayflow_check_done 124 \"$elapsed\" stopped; else sleep 1; if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; else echo \"relayflow: the checks stopped without reporting an exit status.\" >> \"$check_out\"; relayflow_check_done 125 \"$elapsed\"; fi; fi; fi; fi ); rm -f \"$recipe\"; git checkout -q -f \"$orig\" >/dev/null 2>&1 || git checkout -q -f \"$head\" >/dev/null 2>&1; if [ \"$(git rev-parse HEAD 2>/dev/null)\" = \"$head\" ]; then echo \"$token\"; else echo \"relayflow: could not return to $orig after checking the base commit.\" >&2; echo unknown; fi; elif if [ -n \"${recipe:-}\" ]; then rm -f \"$recipe\"; fi; tmp=$(mktemp -d \"${TMPDIR:-/tmp}/relayflow-base.XXXXXX\") && git worktree add --detach -q \"$tmp/base\" \"$base\" >/dev/null 2>&1; then relayflow_build_dirs \"$root\" | while IFS= read -r d; do if [ ! -e \"$tmp/base/$d\" ] && mkdir -p \"$(dirname \"$tmp/base/$d\")\"; then ln -s \"$root/$d\" \"$tmp/base/$d\"; fi; done; check_dir=\"$tmp/base\"; check_out=\"$root/.relayflow/base-check.log\"; root=\"$PWD\"; script=\"${check_script:-$root/.relayflow/check.sh}\"; check_dir=\"${check_dir:-$root}\"; check_out=\"${check_out:-$root/.relayflow/check.log}\"; mkdir -p \"$(dirname \"$check_out\")\"; rm -f \"$check_out.exit\" \"$check_out.elapsed\" \"$check_out.limit\"; if [ ! -s \"$script\" ]; then echo \"relayflow: there is no check script, so no checks ran.\" > \"$check_out\"; echo \"relayflow: there is no check script, so no checks ran.\" >&2; echo none; else limit=\"${RELAYFLOW_CHECK_TIMEOUT:-840}\"; limiter=; if command -v timeout >/dev/null 2>&1; then limiter=timeout; elif command -v gtimeout >/dev/null 2>&1; then limiter=gtimeout; elif command -v perl >/dev/null 2>&1; then limiter=perl; fi; relayflow_limiter='my $l = shift; my $p = fork; defined $p or exit 127; if (!$p) { setpgrp(0, 0); exec @ARGV or exit 127 } $SIG{ALRM} = sub { kill '\\''TERM'\\'', -$p; sleep 2; kill '\\''KILL'\\'', -$p; exit 124 }; alarm $l; waitpid($p, 0); exit($? & 127 ? 128 + ($? & 127) : $? >> 8)'; grouper=; if command -v setsid >/dev/null 2>&1; then grouper=setsid; elif command -v perl >/dev/null 2>&1; then grouper=perl; fi; relayflow_inline=; relayflow_check_done() { printf '%s\\n' \"$1\" > \"$check_out.exit\"; printf '%s\\n' \"$2\" > \"$check_out.elapsed\"; printf '%s\\n' \"$limit\" > \"$check_out.limit\"; tail -n 40 \"$check_out\" >&2; if [ \"$1\" -eq 0 ]; then echo \"relayflow: the checks passed.\" >&2; echo pass; elif [ \"$1\" -eq 124 ] && { { [ \"$relayflow_inline\" = yes ] && [ -n \"$limiter\" ]; } || [ \"${3:-}\" = stopped ]; }; then echo \"relayflow: the checks did not finish within ${limit}s (they ran $2s).\" >&2; echo timeout; else echo \"relayflow: the checks failed with exit $1; the full output is in $check_out.\" >&2; echo fail; fi; }; if [ -z \"$limiter\" ]; then relayflow_unrunnable=\"relayflow: this machine has no timeout, gtimeout or perl, so there is no way to stop the checks at a time limit and they were not run. Install coreutils (timeout) or perl, then run again.\"; printf '%s\\n' \"$relayflow_unrunnable\" > \"$check_out\"; echo \"$relayflow_unrunnable\" >&2; echo unrunnable; elif [ -z \"${RELAYFLOW_CHECK_WAIT:-}\" ] || [ -z \"$grouper\" ]; then relayflow_inline=yes; if [ -n \"${RELAYFLOW_CHECK_WAIT:-}\" ]; then if [ \"$RELAYFLOW_CHECK_WAIT\" -lt \"$limit\" ]; then limit=\"$RELAYFLOW_CHECK_WAIT\"; fi; echo \"relayflow: neither setsid nor perl is installed, so the checks cannot be detached; they run in this step under $limiter, limited to ${limit}s.\" >&2; fi; echo \"relayflow: running $script in $check_dir\" >&2; since=$(date +%s); ( cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; case \"$limiter\" in (timeout|gtimeout) \"$limiter\" \"$limit\" sh \"$script\" ;; (perl) perl -e \"$relayflow_limiter\" \"$limit\" sh \"$script\" ;; (*) sh \"$script\" ;; esac > \"$check_out\" 2>&1 < /dev/null; printf \"%s\\n\" \"$?\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\" ); relayflow_check_done \"$(cat \"$check_out.status\" 2>/dev/null || echo 125)\" \"$(( $(date +%s) - since ))\"; rm -f \"$check_out.status\"; else id=\"${RELAYFLOW_CHECK_ID:-1}\"; relayflow_group() { cat \"$check_out.group\" 2>/dev/null || cat \"$check_out.pid\" 2>/dev/null; }; relayflow_alive() { g=$(relayflow_group); [ -n \"$g\" ] && { kill -0 \"-$g\" 2>/dev/null || kill -0 \"$g\" 2>/dev/null; }; }; relayflow_stop() { g=$(relayflow_group); [ -n \"$g\" ] || return 0; kill -TERM \"-$g\" 2>/dev/null || kill -TERM \"$g\" 2>/dev/null; i=0; while relayflow_alive && [ \"$i\" -lt 5 ]; do sleep 1; i=$((i + 1)); done; kill -KILL \"-$g\" 2>/dev/null || kill -KILL \"$g\" 2>/dev/null; return 0; }; relayflow_status_done() { if [ -f \"$check_out.stopped\" ]; then relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\" stopped; else relayflow_check_done \"$(cat \"$check_out.status\")\" \"$1\"; fi; }; if [ \"$(cat \"$check_out.id\" 2>/dev/null)\" != \"$id\" ]; then if [ ! -f \"$check_out.status\" ] && relayflow_alive; then if ! command -v ps >/dev/null 2>&1; then relayflow_stop; echo \"relayflow: stopped what an earlier attempt left running (no ps here to identify it).\" >&2; elif ps -o args= -p \"$(relayflow_group)\" 2>/dev/null | grep -qF 'check_out.group'; then relayflow_stop; echo \"relayflow: stopped a check an earlier attempt left running.\" >&2; fi; fi; rm -f \"$check_out.status\" \"$check_out.status.tmp\" \"$check_out.stopped\" \"$check_out.group\" \"$check_out.pid\"; : > \"$check_out\"; date +%s > \"$check_out.since\"; printf '%s\\n' \"$id\" > \"$check_out.id\"; export check_dir check_out script limit; if [ \"$grouper\" = setsid ]; then setsid sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & else perl -e '$SIG{HUP} = \"IGNORE\"; setpgrp(0, 0); exec @ARGV or exit 127' sh -c 'printf \"%s\\n\" \"$$\" > \"$check_out.group\"; cd \"$check_dir\" || { printf \"%s\\n\" 126 > \"$check_out.status\"; exit 0; }; unset GIT_CONFIG_COUNT GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM; sh \"$script\" > \"$check_out\" 2>&1 < /dev/null & c=$!; ( s=$(date +%s); while kill -0 \"$c\" 2>/dev/null; do if [ $(( $(date +%s) - s )) -gt $((limit + 60)) ]; then : > \"$check_out.stopped\"; printf \"%s\\n\" 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; kill -KILL \"-$$\" 2>/dev/null || kill -KILL \"$c\" 2>/dev/null; fi; sleep 5; done ) > /dev/null 2>&1 & wait \"$c\"; st=$?; if [ ! -f \"$check_out.stopped\" ]; then printf \"%s\\n\" \"$st\" > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; fi' > /dev/null 2>&1 < /dev/null & fi; echo \"$!\" > \"$check_out.pid\"; echo \"relayflow: started $script in $check_dir; it may run ${limit}s in all, and each step waits up to ${RELAYFLOW_CHECK_WAIT}s for it.\" >&2; fi; since=$(cat \"$check_out.since\" 2>/dev/null || date +%s); waited=0; while [ ! -f \"$check_out.status\" ] && [ \"$waited\" -lt \"$RELAYFLOW_CHECK_WAIT\" ] && relayflow_alive && [ $(( $(date +%s) - since )) -lt \"$limit\" ]; do sleep 1; waited=$((waited + 1)); done; elapsed=$(( $(date +%s) - since )); if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; elif relayflow_alive && [ \"$elapsed\" -lt \"$limit\" ]; then echo \"relayflow: the checks are still running (${elapsed}s of ${limit}s).\" >&2; echo running; elif relayflow_alive; then : > \"$check_out.stopped\"; printf '%s\\n' 124 > \"$check_out.status.tmp\"; mv -f \"$check_out.status.tmp\" \"$check_out.status\"; relayflow_stop; echo \"relayflow: stopped the checks after ${elapsed}s, at their ${limit}s.\" >> \"$check_out\"; relayflow_check_done 124 \"$elapsed\" stopped; else sleep 1; if [ -f \"$check_out.status\" ]; then relayflow_status_done \"$elapsed\"; else echo \"relayflow: the checks stopped without reporting an exit status.\" >> \"$check_out\"; relayflow_check_done 125 \"$elapsed\"; fi; fi; fi; fi; git worktree remove --force \"$tmp/base\" >/dev/null 2>&1; git worktree prune >/dev/null 2>&1; rm -rf \"$tmp\"; else echo \"relayflow: could not check out the base commit to compare against.\" >&2; if [ -n \"$tmp\" ]; then rm -rf \"$tmp\"; fi; echo unknown; fi", baseLimit, baseLimit + 2); + // A base check that ran out of time says nothing about a branch that + // failed outright; only a branch that timed out too compares with it. + return result === "timeout" && check !== "timeout" ? "unknown" : result; + }; + const baseline = await compareWithBase(); + + // Publish the branch and open the pull request without an agent. + // Doing no work is a legitimate outcome: a repository with nothing to act on + // leaves no commits and no summary.md. Pushing a branch at the base commit + // and failing inside "gh pr create" is not the report such a run should + // leave, so the check below decides, and anything it cannot vouch for is + // treated as nothing to publish. Working files an agent committed are taken + // out of the branch first, so they are neither published nor counted. + const dropWorkingFiles = "paths='summary.md plan.md reviewed-plan.md review.md review.clean review-blocked.md comparison.md prototype-notes.md .relayflow START-HERE.txt flow-input.json relay-preflight.mjs software-factory.flow.mts'; if [ -z \"$base\" ] || ! git rev-parse --verify --quiet \"$base^{commit}\" >/dev/null 2>&1; then echo \"relayflow: the base commit is unknown, so working files were left as committed.\" >&2; else drop=$(git ls-files -- $paths | while IFS= read -r p; do git cat-file -e \"$base:$p\" 2>/dev/null || printf '%s\\n' \"$p\"; done); if [ -z \"$drop\" ]; then echo \"relayflow: no working files were committed.\" >&2; else tmp=$(mktemp -d); msg=\"Keep relayflow working files out of the change\"; if GIT_INDEX_FILE=\"$tmp/index\" git read-tree HEAD && printf '%s\\n' \"$drop\" | GIT_INDEX_FILE=\"$tmp/index\" git update-index --force-remove --stdin && tree=$(GIT_INDEX_FILE=\"$tmp/index\" git write-tree) && { commit=$(git commit-tree \"$tree\" -p HEAD -m \"$msg\" 2>/dev/null) || commit=$(git -c user.name=Relayflow -c user.email=noreply@agentrelay.com commit-tree \"$tree\" -p HEAD -m \"$msg\"); } && git update-ref -m \"relayflow: drop working files\" HEAD \"$commit\"; then printf '%s\\n' \"$drop\" | git update-index --force-remove --stdin 2>/dev/null; echo \"relayflow: removed working files from the branch: $(printf '%s ' $drop)\" >&2; else echo \"relayflow: could not remove the working files from the branch.\" >&2; fi; rm -rf \"$tmp\"; fi; fi; echo done"; + // Every push goes through this: if GitHub refuses the branch only because + // it edits .github/workflows/ (the run's token may lack that permission), + // the work is pushed without those edits and the patch goes on the PR. + const pushBranch = "relayflow_push() { wf=.github/workflows; err=$(mktemp \"${TMPDIR:-/tmp}/relayflow-push.XXXXXX\") || { echo \"relayflow push-guard: could not create a temporary file, so the push was not attempted (its output could not be scrubbed of credentials).\" >&2; return 1; }; git push \"$@\" 2>\"$err\"; status=$?; sed -e 's#://[^/@[:space:]]*@#://#g' \"$err\" >&2; if [ \"$status\" -eq 0 ]; then rm -f \"$err\"; return 0; fi; if ! grep -Eq 'refusing to allow a GitHub App to create or update workflow|without .workflows. permission' \"$err\"; then rm -f \"$err\"; return \"$status\"; fi; mb=; if [ -n \"${base:-}\" ] && git rev-parse --verify --quiet \"$base^{commit}\" >/dev/null 2>&1; then mb=$(git merge-base \"$base\" HEAD 2>/dev/null); fi; if [ -z \"$mb\" ]; then echo \"relayflow push-guard: GitHub refused the workflow edits, and the base commit is unknown, so nothing was withheld.\" >&2; rm -f \"$err\"; return \"$status\"; fi; orig=$(git rev-parse HEAD); if [ -z \"$(git rev-list --full-history \"$orig\" \"^$mb\" --not --remotes -- \"$wf\")\" ]; then echo \"relayflow push-guard: no unpushed commit edits $wf, so there is nothing to withhold.\" >&2; rm -f \"$err\"; return \"$status\"; fi; tmp=$(mktemp -d \"${TMPDIR:-/tmp}/relayflow-withhold.XXXXXX\") || { rm -f \"$err\"; return \"$status\"; }; : > \"$tmp/map\"; ok=yes; relayflow_new() { n=$(grep \"^$1 \" \"$tmp/map\" | cut -d\" \" -f2); if [ -n \"$n\" ]; then echo \"$n\"; else echo \"$1\"; fi; }; for c in $(git rev-list --reverse --topo-order \"$orig\" \"^$mb\" --not --remotes); do parents=$(git rev-list --parents -n 1 \"$c\" | cut -s -d\" \" -f2-); p1=; np=; for p in $parents; do q=$(relayflow_new \"$p\"); if [ -z \"$p1\" ]; then p1=$q; fi; np=\"$np -p $q\"; done; if GIT_INDEX_FILE=\"$tmp/index\" git read-tree \"$c\" && { GIT_INDEX_FILE=\"$tmp/index\" git ls-files -z -- \"$wf\" | GIT_INDEX_FILE=\"$tmp/index\" git update-index -z --force-remove --stdin; } && { [ -z \"$p1\" ] || git ls-tree -r --full-tree \"$p1\" -- \"$wf\" | GIT_INDEX_FILE=\"$tmp/index\" git update-index --index-info; } && tree=$(GIT_INDEX_FILE=\"$tmp/index\" git write-tree); then :; else ok=no; break; fi; case \"$parents\" in (*\" \"*) single=no ;; (*) single=yes ;; esac; if [ \"$single\" = yes ] && [ -n \"$p1\" ] && [ \"$tree\" = \"$(git rev-parse \"$p1^{tree}\")\" ]; then echo \"$c $p1\" >> \"$tmp/map\"; continue; fi; git cat-file commit \"$c\" | sed \"1,/^\\$/d\" > \"$tmp/msg\"; new=$(GIT_AUTHOR_NAME=\"$(git show -s --format=%an \"$c\")\" GIT_AUTHOR_EMAIL=\"$(git show -s --format=%ae \"$c\")\" GIT_AUTHOR_DATE=\"$(git show -s --date=raw --format=%ad \"$c\")\" GIT_COMMITTER_NAME=\"$(git show -s --format=%cn \"$c\")\" GIT_COMMITTER_EMAIL=\"$(git show -s --format=%ce \"$c\")\" GIT_COMMITTER_DATE=\"$(git show -s --date=raw --format=%cd \"$c\")\" git commit-tree \"$tree\" $np -F \"$tmp/msg\") || { ok=no; break; }; echo \"$c $new\" >> \"$tmp/map\"; done; new=$(relayflow_new \"$orig\"); if [ \"$ok\" != yes ] || [ \"$new\" = \"$orig\" ]; then echo \"relayflow push-guard: could not withhold the workflow edits.\" >&2; rm -rf \"$tmp\" \"$err\"; return \"$status\"; fi; mkdir -p .relayflow; patch=.relayflow/workflow-changes.patch; section=.relayflow/workflow-changes.md; git diff --full-index \"$mb\" \"$orig\" -- \"$wf\" > \"$patch\"; n=$(git diff --name-only \"$mb\" \"$orig\" -- \"$wf\" | wc -l | tr -d \" \"); relayflow_block() { block_run=$(LC_ALL=C awk '{ s = $0; while (match(s, /`+/)) { if (RLENGTH > m) m = RLENGTH; s = substr(s, RSTART + RLENGTH) } } END { print m + 0 }' \"$1\"); if [ \"${block_run:-0}\" -lt 3 ]; then block_run=2; fi; block_fence=$(printf \"%$((block_run + 1))s\" \"\" | tr \" \" '`'); printf \"%s%s\\n\" \"$block_fence\" \"$2\"; cat \"$1\"; if [ -s \"$1\" ] && [ \"$(LC_ALL=C tail -c 1 \"$1\" | wc -l | tr -d \" \")\" -eq 0 ]; then echo; fi; printf \"%s\\n\" \"$block_fence\"; }; relayflow_cap() { cap_size=$(LC_ALL=C wc -c < \"$1\" | tr -d \" \"); if [ \"$cap_size\" -le \"$2\" ]; then return 0; fi; echo \"relayflow: $1 is $cap_size bytes, more than the $2 it may hold, so it was cut; the full text is $4.\" >&2; cap_note=\"_…truncated to fit GitHub's limit: $cap_size bytes were cut to under $2. The full text is $4._\"; cap_lead=$(printf '[\\300-\\377]'); cap_cont=$(printf '[\\200-\\277]'); cap_keep=$(($2 - 512)); cap_tries=0; while :; do if [ \"$cap_keep\" -le 0 ] || [ \"$cap_tries\" -ge 3 ]; then : > \"$1.cap\"; break; fi; cap_tries=$((cap_tries + 1)); if [ \"$3\" = tail ]; then LC_ALL=C tail -c \"$cap_keep\" \"$1\" > \"$1.cut\"; else LC_ALL=C head -c \"$cap_keep\" \"$1\" > \"$1.cut\"; fi; cap_lines=$(LC_ALL=C tr -cd \"\\n\" < \"$1.cut\" | wc -c | tr -d \" \"); if [ \"$3\" = tail ]; then { printf '%s\\n\\n' \"$cap_note\"; if [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed 1d \"$1.cut\"; else LC_ALL=C sed \"s/^$cap_cont*//\" \"$1.cut\"; fi; } > \"$1.cap\"; else { if [ \"$(LC_ALL=C tail -c 1 \"$1.cut\" | wc -l | tr -d \" \")\" -eq 1 ]; then cat \"$1.cut\"; elif [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed '$d' \"$1.cut\"; else LC_ALL=C sed \"s/$cap_lead$cap_cont*\\$//\" \"$1.cut\"; echo; fi; } > \"$1.cap\"; { relayflow_block \"$1.cap\" text; printf \"\\n%s\\n\" \"$cap_note\"; } > \"$1.cut\"; mv -f \"$1.cut\" \"$1.cap\"; fi; cap_out=$(LC_ALL=C wc -c < \"$1.cap\" | tr -d \" \"); if [ \"$cap_out\" -le \"$2\" ]; then break; fi; cap_keep=$((cap_keep - cap_out + $2)); done; rm -f \"$1.cut\"; mv -f \"$1.cap\" \"$1\"; return 0; }; relayflow_section() { used=0; if [ \"${comment:-}\" != yes ] && [ -f .relayflow/pr-body.md ]; then used=$(wc -c < .relayflow/pr-body.md | tr -d \" \"); fi; { printf '\\n## Workflow changes not applied\\n\\n%s\\n\\n' \"The GitHub App token this run pushes with lacks the \\`workflows\\` permission, so GitHub refused the commits that change \\`.github/workflows/\\`. The rest of the work is pushed; these edits were taken out of its commits. Apply them manually:\"; git diff --name-status \"$mb\" \"$orig\" -- \"$wf\" | awk -F '\\t' '{ printf \"- %s \\140%s\\140\\n\", substr($1, 1, 1), $NF }' | head -n 50; if [ \"$n\" -gt 50 ]; then printf -- '- …and %s more\\n' \"$((n - 50))\"; fi; } > \"$tmp/head\"; overhead=$(( $(wc -c < \"$tmp/head\" | tr -d \" \") + 400 )); limit=${RELAYFLOW_WITHHELD_PATCH_LIMIT:-61440}; room=$((60000 - used - overhead)); if [ \"$room\" -lt \"$limit\" ]; then limit=$room; fi; size=$(wc -c < \"$patch\" | tr -d \" \"); { cat \"$tmp/head\"; if [ \"$limit\" -le 0 ]; then printf '\\n%s\\n' \"_The patch ($size bytes) does not fit in the pull request body. The full patch is .relayflow/workflow-changes.patch in the run workspace, and in this push step's output._\"; elif [ \"$size\" -le \"$limit\" ]; then printf '\\n````diff\\n'; cat \"$patch\"; printf '````\\n'; else printf '\\n````diff\\n'; head -c \"$limit\" \"$patch\" | sed '$d'; printf '````\\n\\n%s\\n' \"_Truncated to $limit of $size bytes. The full patch is .relayflow/workflow-changes.patch in the run workspace, and in this push step's output._\"; fi; } > \"$section\"; relayflow_cap \"$section\" $((60000 - used)) head \"$patch in the run workspace\"; }; dirty=$( { git diff --name-only \"$orig\" -- \"$wf\"; git diff --cached --name-only \"$orig\" -- \"$wf\"; git ls-files --others --exclude-standard -- \"$wf\"; } 2>/dev/null | sort -u); if [ -z \"$(git rev-list \"$new\" --not --remotes 2>/dev/null)\" ]; then relayflow_section; cat \"$patch\" >&2; if [ \"${comment:-}\" = yes ] && { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change comment --body-file \"$section\"; else gh pr comment --body-file \"$section\"; fi; } >/dev/null 2>&1; then echo \"relayflow push-guard: every new change edits $wf, which this token cannot push, so nothing else was pushed; posted the withheld workflow changes to the pull request.\" >&2; rm -rf \"$tmp\" \"$err\"; return 0; fi; echo \"relayflow push-guard: every change in this run edits $wf, which this token cannot push, so there is nothing else to publish. The edits are in $patch and printed above.\" >&2; rm -rf \"$tmp\" \"$err\"; return \"$status\"; fi; git update-ref -m \"relayflow: withhold workflow edits\" HEAD \"$new\" \"$orig\"; git push \"$@\" 2>\"$tmp/push\"; again=$?; sed -e 's#://[^/@[:space:]]*@#://#g' \"$tmp/push\" >&2; if [ \"$again\" -ne 0 ]; then git update-ref -m \"relayflow: restore after a failed push\" HEAD \"$orig\" \"$new\"; echo \"relayflow push-guard: the push failed again after withholding the workflow edits. The original error was:\" >&2; sed -e 's#://[^/@[:space:]]*@#://#g' \"$err\" >&2; rm -rf \"$tmp\" \"$err\"; return \"$again\"; fi; previous=$(git rev-parse --verify --quiet refs/relayflow/withheld-workflows 2>/dev/null || :); if [ -n \"$previous\" ] && [ \"$previous\" != \"$orig\" ]; then git update-ref \"refs/relayflow/withheld-workflows-history/$previous\" \"$previous\"; fi; git update-ref refs/relayflow/withheld-workflows \"$orig\"; git diff --name-only --no-renames \"$new\" \"$orig\" -- \"$wf\" | while IFS= read -r p; do if printf \"%s\\n\" \"$dirty\" | grep -qxF -- \"$p\"; then echo \"relayflow push-guard: $p has uncommitted edits, so it was left as it is.\" >&2; continue; fi; if git cat-file -e \"$new:$p\" 2>/dev/null; then git checkout -q \"$new\" -- \"$p\"; else git rm -q -f --ignore-unmatch -- \"$p\" >/dev/null 2>&1; rm -f -- \"$p\"; fi; done; if [ -s \"$patch\" ]; then relayflow_section; if [ \"$size\" -gt \"$limit\" ]; then cat \"$patch\" >&2; fi; if [ \"${comment:-}\" = yes ]; then if { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change comment --body-file \"$section\"; else gh pr comment --body-file \"$section\"; fi; } >/dev/null 2>&1; then echo \"relayflow push-guard: posted the withheld workflow changes to the pull request.\" >&2; else echo \"relayflow push-guard: could not comment on the pull request; $section holds the withheld workflow changes.\" >&2; fi; elif [ -f .relayflow/pr-body.md ]; then cat \"$section\" >> .relayflow/pr-body.md; fi; fi; echo \"relayflow push-guard: workflow edits withheld ($n files)\"; echo \"relayflow push-guard: GitHub refused the edits to $wf, so the branch was pushed without them. The patch is $patch; the original commits are refs/relayflow/withheld-workflows.\" >&2; rm -rf \"$tmp\" \"$err\"; }; relayflow_push"; + await f.run("base=" + baseCommit + "; " + dropWorkingFiles, { timeout: "5m" }); + const publishCheck = "if [ -s summary.md ]; then summary=yes; else summary=no; fi; changed=unknown; if [ -n \"$base\" ] && git rev-parse --verify --quiet \"$base^{commit}\" >/dev/null 2>&1; then if git diff --quiet \"$base\" HEAD 2>/dev/null; then changed=no; else changed=yes; fi; fi; if [ \"$changed\" != no ] && [ \"$summary\" = yes ]; then echo \"relayflow: new commits and a summary.md are present; opening the pull request.\" >&2 && echo publish && exit 0; fi; if [ \"$changed\" = yes ]; then echo \"relayflow: there are commits but summary.md is missing or empty, so there is no pull-request body.\" >&2 && echo no-summary && exit 0; fi; echo \"relayflow: no commits were made on this branch, so there is nothing to push or publish.\" >&2 && echo no-commits && exit 0"; + const publish = (await f.run("base=" + baseCommit + "; " + publishCheck)).trim(); + if (publish !== "publish" && publish !== "no-summary") { + console.error("Stopped: the agents made no commits on this branch, so there is nothing to publish. No branch was pushed and no pull request was opened."); + return f.done("needs_human"); + } + if (publish !== "publish") { + await f.run("base=" + baseCommit + "; " + pushBranch + " --set-upstream origin HEAD", { timeout: "11m" }); + console.error("Stopped: the branch was pushed, but no summary.md was written, so there is no pull-request body. Open the pull request by hand, or run again."); + return f.done("needs_human"); + } + // Failing, unrun or unfinished work is never thrown away: the pull request + // opens as a draft, with the verdict, the script and the output in its body. + const checkReport = "mkdir -p .relayflow; relayflow_block() { block_run=$(LC_ALL=C awk '{ s = $0; while (match(s, /`+/)) { if (RLENGTH > m) m = RLENGTH; s = substr(s, RSTART + RLENGTH) } } END { print m + 0 }' \"$1\"); if [ \"${block_run:-0}\" -lt 3 ]; then block_run=2; fi; block_fence=$(printf \"%$((block_run + 1))s\" \"\" | tr \" \" '`'); printf \"%s%s\\n\" \"$block_fence\" \"$2\"; cat \"$1\"; if [ -s \"$1\" ] && [ \"$(LC_ALL=C tail -c 1 \"$1\" | wc -l | tr -d \" \")\" -eq 0 ]; then echo; fi; printf \"%s\\n\" \"$block_fence\"; }; relayflow_cap() { cap_size=$(LC_ALL=C wc -c < \"$1\" | tr -d \" \"); if [ \"$cap_size\" -le \"$2\" ]; then return 0; fi; echo \"relayflow: $1 is $cap_size bytes, more than the $2 it may hold, so it was cut; the full text is $4.\" >&2; cap_note=\"_…truncated to fit GitHub's limit: $cap_size bytes were cut to under $2. The full text is $4._\"; cap_lead=$(printf '[\\300-\\377]'); cap_cont=$(printf '[\\200-\\277]'); cap_keep=$(($2 - 512)); cap_tries=0; while :; do if [ \"$cap_keep\" -le 0 ] || [ \"$cap_tries\" -ge 3 ]; then : > \"$1.cap\"; break; fi; cap_tries=$((cap_tries + 1)); if [ \"$3\" = tail ]; then LC_ALL=C tail -c \"$cap_keep\" \"$1\" > \"$1.cut\"; else LC_ALL=C head -c \"$cap_keep\" \"$1\" > \"$1.cut\"; fi; cap_lines=$(LC_ALL=C tr -cd \"\\n\" < \"$1.cut\" | wc -c | tr -d \" \"); if [ \"$3\" = tail ]; then { printf '%s\\n\\n' \"$cap_note\"; if [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed 1d \"$1.cut\"; else LC_ALL=C sed \"s/^$cap_cont*//\" \"$1.cut\"; fi; } > \"$1.cap\"; else { if [ \"$(LC_ALL=C tail -c 1 \"$1.cut\" | wc -l | tr -d \" \")\" -eq 1 ]; then cat \"$1.cut\"; elif [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed '$d' \"$1.cut\"; else LC_ALL=C sed \"s/$cap_lead$cap_cont*\\$//\" \"$1.cut\"; echo; fi; } > \"$1.cap\"; { relayflow_block \"$1.cap\" text; printf \"\\n%s\\n\" \"$cap_note\"; } > \"$1.cut\"; mv -f \"$1.cut\" \"$1.cap\"; fi; cap_out=$(LC_ALL=C wc -c < \"$1.cap\" | tr -d \" \"); if [ \"$cap_out\" -le \"$2\" ]; then break; fi; cap_keep=$((cap_keep - cap_out + $2)); done; rm -f \"$1.cut\"; mv -f \"$1.cap\" \"$1\"; return 0; }; report=.relayflow/check-report.md; parts=.relayflow/report-parts; rm -rf \"$parts\"; mkdir -p \"$parts\"; if [ -s .relayflow/check.sh ]; then cp .relayflow/check.sh \"$parts/script\"; relayflow_cap \"$parts/script\" 4000 head \".relayflow/check.sh in the run workspace\"; fi; if [ -s .relayflow/check.log ]; then tail -n 80 .relayflow/check.log > \"$parts/check\"; relayflow_cap \"$parts/check\" 10000 tail \".relayflow/check.log in the run workspace\"; fi; if [ -s .relayflow/base-check.log ]; then tail -n 80 .relayflow/base-check.log > \"$parts/base\"; relayflow_cap \"$parts/base\" 10000 tail \".relayflow/base-check.log in the run workspace\"; fi; if [ -s .relayflow/repair-notes.md ]; then cp .relayflow/repair-notes.md \"$parts/repair\"; relayflow_cap \"$parts/repair\" 4000 head \".relayflow/repair-notes.md in the run workspace\"; fi; relayflow_number() { v=$(cat \"$1\" 2>/dev/null | head -n 1); case \"$v\" in (\"\"|*[!0-9]*) ;; (*) printf %s \"$v\" ;; esac; }; relayflow_duration() { if [ \"$1\" -ge 60 ]; then printf \"%dm%02ds\" $(($1 / 60)) $(($1 % 60)); else printf \"%ds\" \"$1\"; fi; }; ck_exit=$(relayflow_number .relayflow/check.log.exit); ck_elapsed=$(relayflow_number .relayflow/check.log.elapsed); ck_limit=$(relayflow_number .relayflow/check.log.limit); { printf '## Checks\\n\\n'; if [ \"${implementer_timeout:-}\" = yes ]; then printf '%s\\n\\n' \"**The implementer was stopped at its time limit**, so this change may be incomplete. Its committed work is what was checked and pushed; the pull request is a draft.\"; fi; case \"$check\" in pass) printf '%s\\n' \"Relayflow ran this repository's checks (.relayflow/check.sh) and they passed.\" ;; skipped) printf '%s\\n' \"**Relayflow ran out of time before it could run this repository's checks** (.relayflow/check.sh), so no checks ran on this change. The pull request is a draft; run the checks before merging.\" ;; unrunnable) printf '%s\\n' \"**Relayflow could not run this repository's checks** (.relayflow/check.sh): the machine it ran on has no \\`timeout\\`, \\`gtimeout\\` or \\`perl\\`, so there was no way to stop the suite at a time limit, and starting it could have ended the run with no verdict at all. This is not a test failure, and nothing here says the change is broken. Install coreutils (for \\`timeout\\`) or perl and run the flow again, or run .relayflow/check.sh by hand. The pull request is a draft; run the checks before merging.\" ;; none) printf '%s\\n' \"Relayflow found no way to run this repository's tests — no check command, no CI test job it could follow, no test target and no recognised project file — so no checks ran. Review the change with that in mind.\" ;; *) if [ \"$check\" = timeout ]; then if [ -n \"$ck_elapsed\" ] && [ -n \"$ck_limit\" ]; then printf '%s\\n\\n' \"**The checks timed out**: they ran for $(relayflow_duration \"$ck_elapsed\") of their $(relayflow_duration \"$ck_limit\") budget and were stopped before they finished, so this is a time limit, not a test failure. Run them locally (\\`sh .relayflow/check.sh\\`) to see how long they take, or raise the check budget (\\`checkTotal\\` in the flow) and run the flow again.\"; else printf '%s\\n\\n' \"**The checks timed out**: they were stopped at their time budget before they finished, so this is a time limit, not a test failure. Run them locally (\\`sh .relayflow/check.sh\\`) to see how long they take, or raise the check budget (\\`checkTotal\\` in the flow) and run the flow again.\"; fi; else if [ -n \"$ck_exit\" ]; then printf '%s\\n\\n' \"**The checks failed** (exit $ck_exit). The end of their output is below.\"; else printf '%s\\n\\n' \"**The checks failed.** The end of their output is below.\"; fi; fi; case \"$baseline\" in fail|timeout) printf '%s\\n' \"**The checks fail on the base commit too**, so this was not introduced by this change: it comes from the repository itself or from the environment the checks ran in. This pull request is a draft until someone looks.\" ;; pass) printf '%s\\n' \"**This change breaks checks that pass on the base commit.** The flow tried to repair it and could not. The pull request is a draft so the work is not lost; it is not ready to merge.\" ;; new) printf '%s\\n' \"**The checks this change adds fail.** The repository had no checks before this change, so there is no base commit to compare with; the flow tried to repair them and could not. The pull request is a draft so the work is not lost; it is not ready to merge.\" ;; skipped) printf '%s\\n' \"There was no time left in the run to check the base commit (base not checked), so it is not known whether this change caused this. This pull request is a draft until someone looks.\" ;; *) printf '%s\\n' \"The base commit could not be checked for comparison, so it is not known whether this change caused this. This pull request is a draft until someone looks.\" ;; esac ;; esac; if [ -s \"$parts/script\" ]; then printf '\\n
What ran (.relayflow/check.sh)\\n\\n'; relayflow_block \"$parts/script\" sh; printf '
\\n'; fi; if [ \"$check\" != pass ] && [ \"$check\" != none ] && [ \"$check\" != skipped ] && [ \"$check\" != unrunnable ] && [ -s \"$parts/check\" ]; then printf '\\n
Output on this branch (last 80 lines)\\n\\n'; relayflow_block \"$parts/check\" ''; printf '
\\n'; fi; if [ \"$baseline\" = fail ] || [ \"$baseline\" = timeout ] || [ \"$baseline\" = unknown ]; then if [ -s \"$parts/base\" ]; then printf '\\n
Output on the base commit (last 80 lines)\\n\\n'; relayflow_block \"$parts/base\" ''; printf '
\\n'; fi; fi; if [ -s \"$parts/repair\" ]; then printf '\\n### What the repair agent found\\n\\n'; cat \"$parts/repair\"; fi; } > \"$report\"; if [ -s summary.md ]; then room=$((57888 - $(LC_ALL=C wc -c < \"$report\" | tr -d \" \") - $(printf %s \"${reference:-}\" | LC_ALL=C wc -c | tr -d \" \"))); if [ -n \"${reference:-}\" ] && [ \"$(LC_ALL=C wc -c < summary.md | tr -d \" \")\" -gt \"$room\" ]; then grep -vxF -- \"$reference\" summary.md > \"$parts/summary\" || :; else cp summary.md \"$parts/summary\"; fi; relayflow_cap \"$parts/summary\" \"$room\" head \"summary.md in the run workspace\"; fi; { if [ -s \"$parts/summary\" ]; then cat \"$parts/summary\"; printf '\\n\\n'; fi; cat \"$report\"; } > .relayflow/pr-body.md; rm -rf \"$parts\"; echo \"relayflow: wrote the check report to $report.\" >&2; echo written"; + await f.run("check=" + check + "; baseline=" + verdictOf(baseline) + "; implementer_timeout=" + (implementerTimedOut ? "yes" : "no") + "; reference=" + shellQuote(changeReference) + "; " + checkReport); + const prepareChangeMetadata = "if [ ! -s .relayflow/pr-body.md ]; then echo missing-body; exit 0; fi; relayflow_block() { block_run=$(LC_ALL=C awk '{ s = $0; while (match(s, /`+/)) { if (RLENGTH > m) m = RLENGTH; s = substr(s, RSTART + RLENGTH) } } END { print m + 0 }' \"$1\"); if [ \"${block_run:-0}\" -lt 3 ]; then block_run=2; fi; block_fence=$(printf \"%$((block_run + 1))s\" \"\" | tr \" \" '`'); printf \"%s%s\\n\" \"$block_fence\" \"$2\"; cat \"$1\"; if [ -s \"$1\" ] && [ \"$(LC_ALL=C tail -c 1 \"$1\" | wc -l | tr -d \" \")\" -eq 0 ]; then echo; fi; printf \"%s\\n\" \"$block_fence\"; }; relayflow_cap() { cap_size=$(LC_ALL=C wc -c < \"$1\" | tr -d \" \"); if [ \"$cap_size\" -le \"$2\" ]; then return 0; fi; echo \"relayflow: $1 is $cap_size bytes, more than the $2 it may hold, so it was cut; the full text is $4.\" >&2; cap_note=\"_…truncated to fit GitHub's limit: $cap_size bytes were cut to under $2. The full text is $4._\"; cap_lead=$(printf '[\\300-\\377]'); cap_cont=$(printf '[\\200-\\277]'); cap_keep=$(($2 - 512)); cap_tries=0; while :; do if [ \"$cap_keep\" -le 0 ] || [ \"$cap_tries\" -ge 3 ]; then : > \"$1.cap\"; break; fi; cap_tries=$((cap_tries + 1)); if [ \"$3\" = tail ]; then LC_ALL=C tail -c \"$cap_keep\" \"$1\" > \"$1.cut\"; else LC_ALL=C head -c \"$cap_keep\" \"$1\" > \"$1.cut\"; fi; cap_lines=$(LC_ALL=C tr -cd \"\\n\" < \"$1.cut\" | wc -c | tr -d \" \"); if [ \"$3\" = tail ]; then { printf '%s\\n\\n' \"$cap_note\"; if [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed 1d \"$1.cut\"; else LC_ALL=C sed \"s/^$cap_cont*//\" \"$1.cut\"; fi; } > \"$1.cap\"; else { if [ \"$(LC_ALL=C tail -c 1 \"$1.cut\" | wc -l | tr -d \" \")\" -eq 1 ]; then cat \"$1.cut\"; elif [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed '$d' \"$1.cut\"; else LC_ALL=C sed \"s/$cap_lead$cap_cont*\\$//\" \"$1.cut\"; echo; fi; } > \"$1.cap\"; { relayflow_block \"$1.cap\" text; printf \"\\n%s\\n\" \"$cap_note\"; } > \"$1.cut\"; mv -f \"$1.cut\" \"$1.cap\"; fi; cap_out=$(LC_ALL=C wc -c < \"$1.cap\" | tr -d \" \"); if [ \"$cap_out\" -le \"$2\" ]; then break; fi; cap_keep=$((cap_keep - cap_out + $2)); done; rm -f \"$1.cut\"; mv -f \"$1.cap\" \"$1\"; return 0; }; relayflow_reference() { if [ -n \"$reference\" ] && [ \"$(printf %s \"$reference\" | LC_ALL=C wc -c | tr -d \" \")\" -le 1024 ] && ! grep -qxF \"$reference\" .relayflow/pr-body.md; then printf \"\\n%s\\n\" \"$reference\" >> .relayflow/pr-body.md; fi; }; relayflow_reference; if [ \"$(LC_ALL=C wc -c < .relayflow/pr-body.md | tr -d \" \")\" -gt 57952 ]; then if [ -n \"$reference\" ]; then grep -vxF -- \"$reference\" .relayflow/pr-body.md > .relayflow/pr-body.md.tmp || :; mv -f .relayflow/pr-body.md.tmp .relayflow/pr-body.md; fi; relayflow_cap .relayflow/pr-body.md $((57952 - 2 - $(printf %s \"$reference\" | LC_ALL=C wc -c | tr -d \" \"))) head \"the run journal\"; relayflow_reference; fi; echo prepared"; + await f.run("reference=" + shellQuote(changeReference) + "; " + prepareChangeMetadata); + const validateChangeMetadata = "if [ ! -s .relayflow/pr-body.md ]; then echo missing-body; elif [ \"$(printf %s \"${reference:-}\" | LC_ALL=C wc -c | tr -d \" \")\" -gt 1024 ]; then echo reference-too-long; elif [ -z \"$title\" ]; then echo empty-title; elif ! printf \"%s\\n\" \"$title_length\" | grep -Eq \"^[0-9]+$\"; then echo malformed-title-length; elif [ \"$title_length\" -gt 240 ]; then echo title-too-long; elif [ \"$(printf %s \"$title\" | tr \"[:upper:]\" \"[:lower:]\")\" = \"software factory change\" ] || [ \"$(printf %s \"$title\" | tr \"[:upper:]\" \"[:lower:]\")\" = \"replace with your ticket title\" ]; then echo placeholder-title; elif [ \"$source\" = github ] && ! printf \"%s\\n\" \"$identifier\" | grep -Eq \"^#[1-9][0-9]*$\"; then echo malformed-github-identifier; elif [ \"$source\" = github ]; then expected=\"Fixes $identifier\"; count=$(grep -xcF \"$expected\" .relayflow/pr-body.md || true); if [ \"$count\" -eq 0 ]; then echo missing-github-closing-reference; elif [ \"$count\" -ne 1 ]; then echo duplicate-github-closing-reference; else echo valid; fi; else echo valid; fi"; + const metadataVerdict = (await f.run("reference=" + shellQuote(changeReference) + "; title=" + shellQuote(changeTitle) + "; title_length=" + changeTitleLength + "; source=" + shellQuote(issueSource) + "; identifier=" + shellQuote(issueIdentifier) + "; " + validateChangeMetadata)).trim(); + if (metadataVerdict !== "valid") { + console.error("Stopped: invalid pull-request metadata (" + metadataVerdict + "). No branch was pushed and no pull request was opened."); + return f.done("needs_human"); + } + await f.run("base=" + baseCommit + "; " + pushBranch + " --set-upstream origin HEAD", { timeout: "11m" }); + // Hosted runs put relayflow-open-change on PATH: gh pr create on GitHub, a + // merge request on GitLab. A local run has only gh. + const openChange = "open_change() { if command -v relayflow-open-change >/dev/null 2>&1; then relayflow-open-change \"$@\"; else gh pr create \"$@\"; fi; }; open_change"; + const draft = broken(check) || check === "skipped" || check === "unrunnable" || implementerTimedOut; + await f.run(openChange + " --title " + shellQuote(changeTitle) + " --body-file .relayflow/pr-body.md" + (draft ? " --draft" : ""), { timeout: "5m" }); + if (broken(check) && (baseline === "pass" || baseline === "new")) { + // The base commit passes and this branch does not, or the checks are the + // change's own and fail: the change broke them and repair could not fix + // it. That is this flow's verdict on its own work, the same as a failed + // review, so it reports step_failed. + console.error(baseline === "new" + ? "Stopped: the checks this change adds fail, and repair could not fix them. The pull request is a draft with the output." + : "Stopped: this change breaks checks that pass on the base commit, and repair could not fix it. The pull request is a draft with the output."); + return f.done("step_failed"); + } + if (broken(check)) { + // The base commit fails too, or could not be checked: nothing here says the + // change is at fault, so the review still runs and a person decides. + const outcome = check === "timeout" ? "The checks timed out (stopped at their time budget, not a test failure)" : "The checks fail"; + console.error(baseline === "skipped" + ? outcome + ", and there was no time left to check the base commit (base not checked), so it is not known whether this change caused it. The pull request is a draft with the output." + : baseline === "fail" || baseline === "timeout" + ? outcome + ", but not because of this change as far as the base commit shows. The pull request is a draft with the output of both." + : outcome + ", and the base commit could not be checked for comparison, so it is not known whether this change caused it. The pull request is a draft with the output."); + } + + // One independent adversarial review. There is no fix round: what it finds goes on the pull request. + // A review that found problems is this flow's verdict on its own work, so it + // reports done("step_failed"). The pinned runtime (2.0.15 and later) lowers that reason + // (AgentWorkforce/flows#436) and the CLI gives it exit 1, distinct from the + // exit 3 a clean run parks with. An exit code cannot carry what the reviewer + // found, so the step below still drafts the pull request and posts the + // findings to it. + const reviewBlockedCommand = "set -e; { printf '%s\\n\\n' \"**Relayflow: the adversarial review did not pass.** This branch is not approved: the flow stopped here and did not mark it ready to merge.\"; if [ \"${review_timeout:-}\" = yes ]; then if [ -n \"${review_limit:-}\" ]; then stopped_at=\"${review_limit}-minute limit\"; else stopped_at=\"time limit\"; fi; printf 'The review was stopped at its %s before it finished, so it is not a clean review. What it wrote before then is below.\\n\\n' \"$stopped_at\"; fi; if [ -s review.md ]; then cat review.md; else printf '%s\\n' \"_The reviewer left no review.md; see the review step in the run journal._\"; fi; } > review-blocked.md || true; relayflow_block() { block_run=$(LC_ALL=C awk '{ s = $0; while (match(s, /`+/)) { if (RLENGTH > m) m = RLENGTH; s = substr(s, RSTART + RLENGTH) } } END { print m + 0 }' \"$1\"); if [ \"${block_run:-0}\" -lt 3 ]; then block_run=2; fi; block_fence=$(printf \"%$((block_run + 1))s\" \"\" | tr \" \" '`'); printf \"%s%s\\n\" \"$block_fence\" \"$2\"; cat \"$1\"; if [ -s \"$1\" ] && [ \"$(LC_ALL=C tail -c 1 \"$1\" | wc -l | tr -d \" \")\" -eq 0 ]; then echo; fi; printf \"%s\\n\" \"$block_fence\"; }; relayflow_cap() { cap_size=$(LC_ALL=C wc -c < \"$1\" | tr -d \" \"); if [ \"$cap_size\" -le \"$2\" ]; then return 0; fi; echo \"relayflow: $1 is $cap_size bytes, more than the $2 it may hold, so it was cut; the full text is $4.\" >&2; cap_note=\"_…truncated to fit GitHub's limit: $cap_size bytes were cut to under $2. The full text is $4._\"; cap_lead=$(printf '[\\300-\\377]'); cap_cont=$(printf '[\\200-\\277]'); cap_keep=$(($2 - 512)); cap_tries=0; while :; do if [ \"$cap_keep\" -le 0 ] || [ \"$cap_tries\" -ge 3 ]; then : > \"$1.cap\"; break; fi; cap_tries=$((cap_tries + 1)); if [ \"$3\" = tail ]; then LC_ALL=C tail -c \"$cap_keep\" \"$1\" > \"$1.cut\"; else LC_ALL=C head -c \"$cap_keep\" \"$1\" > \"$1.cut\"; fi; cap_lines=$(LC_ALL=C tr -cd \"\\n\" < \"$1.cut\" | wc -c | tr -d \" \"); if [ \"$3\" = tail ]; then { printf '%s\\n\\n' \"$cap_note\"; if [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed 1d \"$1.cut\"; else LC_ALL=C sed \"s/^$cap_cont*//\" \"$1.cut\"; fi; } > \"$1.cap\"; else { if [ \"$(LC_ALL=C tail -c 1 \"$1.cut\" | wc -l | tr -d \" \")\" -eq 1 ]; then cat \"$1.cut\"; elif [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed '$d' \"$1.cut\"; else LC_ALL=C sed \"s/$cap_lead$cap_cont*\\$//\" \"$1.cut\"; echo; fi; } > \"$1.cap\"; { relayflow_block \"$1.cap\" text; printf \"\\n%s\\n\" \"$cap_note\"; } > \"$1.cut\"; mv -f \"$1.cut\" \"$1.cap\"; fi; cap_out=$(LC_ALL=C wc -c < \"$1.cap\" | tr -d \" \"); if [ \"$cap_out\" -le \"$2\" ]; then break; fi; cap_keep=$((cap_keep - cap_out + $2)); done; rm -f \"$1.cut\"; mv -f \"$1.cap\" \"$1\"; return 0; }; relayflow_cap review-blocked.md 60000 head \"review.md in the run workspace\" || true; echo \"relayflow: the adversarial review did not pass; wrote review-blocked.md.\"; if { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change draft; else gh pr ready --undo; fi; } >/dev/null 2>&1; then echo \"relayflow: converted the pull request to a draft.\"; else echo \"relayflow: could not convert the pull request to a draft; review-blocked.md still holds the findings.\" >&2; fi; if { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change comment --body-file review-blocked.md; else gh pr comment --body-file review-blocked.md; fi; } >/dev/null 2>&1; then echo \"relayflow: posted the unresolved review to the pull request.\"; else echo \"relayflow: could not comment on the pull request; review-blocked.md still holds the findings.\" >&2; fi"; + const reportReviewFindingsCommand = "export LC_ALL=C; findings() { tr -d '\\000-\\010\\013-\\037\\177' < review.md | sed '/^[[:space:]]*$/d'; }; size=0; if [ -s review.md ]; then size=$(findings | wc -c | tr -d \" \"); fi; if [ \"$size\" -eq 0 ]; then echo \"relayflow report-review-findings: review.clean absent and review.md missing or empty\" && exit 0; fi; echo \"relayflow report-review-findings: review.clean absent; remaining findings from review.md:\"; findings | head -c 1700; echo; if [ \"$size\" -gt 1700 ]; then echo \"relayflow report-review-findings: cut at 1700 of $size bytes; the full review is in review-blocked.md.\"; fi; exit 0"; + // The pull request is open by now. A review that cannot finish in time + // would leave it looking ready; stop with it as a draft instead. + const reviewLimit = await allowance(20, 2.5); + if (reviewLimit < 8) { + await f.run("mkdir -p .relayflow; printf '%s\\n' \"**This run ran out of time before its review was done** (the flow's time budget). The work is pushed and this pull request is a draft so it is not lost; review it by hand or run the flow again.\" > .relayflow/time-stop.md; relayflow_block() { block_run=$(LC_ALL=C awk '{ s = $0; while (match(s, /`+/)) { if (RLENGTH > m) m = RLENGTH; s = substr(s, RSTART + RLENGTH) } } END { print m + 0 }' \"$1\"); if [ \"${block_run:-0}\" -lt 3 ]; then block_run=2; fi; block_fence=$(printf \"%$((block_run + 1))s\" \"\" | tr \" \" '`'); printf \"%s%s\\n\" \"$block_fence\" \"$2\"; cat \"$1\"; if [ -s \"$1\" ] && [ \"$(LC_ALL=C tail -c 1 \"$1\" | wc -l | tr -d \" \")\" -eq 0 ]; then echo; fi; printf \"%s\\n\" \"$block_fence\"; }; relayflow_cap() { cap_size=$(LC_ALL=C wc -c < \"$1\" | tr -d \" \"); if [ \"$cap_size\" -le \"$2\" ]; then return 0; fi; echo \"relayflow: $1 is $cap_size bytes, more than the $2 it may hold, so it was cut; the full text is $4.\" >&2; cap_note=\"_…truncated to fit GitHub's limit: $cap_size bytes were cut to under $2. The full text is $4._\"; cap_lead=$(printf '[\\300-\\377]'); cap_cont=$(printf '[\\200-\\277]'); cap_keep=$(($2 - 512)); cap_tries=0; while :; do if [ \"$cap_keep\" -le 0 ] || [ \"$cap_tries\" -ge 3 ]; then : > \"$1.cap\"; break; fi; cap_tries=$((cap_tries + 1)); if [ \"$3\" = tail ]; then LC_ALL=C tail -c \"$cap_keep\" \"$1\" > \"$1.cut\"; else LC_ALL=C head -c \"$cap_keep\" \"$1\" > \"$1.cut\"; fi; cap_lines=$(LC_ALL=C tr -cd \"\\n\" < \"$1.cut\" | wc -c | tr -d \" \"); if [ \"$3\" = tail ]; then { printf '%s\\n\\n' \"$cap_note\"; if [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed 1d \"$1.cut\"; else LC_ALL=C sed \"s/^$cap_cont*//\" \"$1.cut\"; fi; } > \"$1.cap\"; else { if [ \"$(LC_ALL=C tail -c 1 \"$1.cut\" | wc -l | tr -d \" \")\" -eq 1 ]; then cat \"$1.cut\"; elif [ \"$cap_lines\" -gt 0 ]; then LC_ALL=C sed '$d' \"$1.cut\"; else LC_ALL=C sed \"s/$cap_lead$cap_cont*\\$//\" \"$1.cut\"; echo; fi; } > \"$1.cap\"; { relayflow_block \"$1.cap\" text; printf \"\\n%s\\n\" \"$cap_note\"; } > \"$1.cut\"; mv -f \"$1.cut\" \"$1.cap\"; fi; cap_out=$(LC_ALL=C wc -c < \"$1.cap\" | tr -d \" \"); if [ \"$cap_out\" -le \"$2\" ]; then break; fi; cap_keep=$((cap_keep - cap_out + $2)); done; rm -f \"$1.cut\"; mv -f \"$1.cap\" \"$1\"; return 0; }; relayflow_cap .relayflow/time-stop.md 60000 head \"the run journal\"; if { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change draft; else gh pr ready --undo; fi; } >/dev/null 2>&1; then echo \"relayflow: converted the pull request to a draft.\"; else echo \"relayflow: could not convert the pull request to a draft.\" >&2; fi; if { if command -v relayflow-change >/dev/null 2>&1; then relayflow-change comment --body-file .relayflow/time-stop.md; else gh pr comment --body-file .relayflow/time-stop.md; fi; } >/dev/null 2>&1; then echo \"relayflow: posted the time stop to the pull request.\"; else echo \"relayflow: could not comment on the pull request.\" >&2; fi", { timeout: "1m" }); + console.error("Stopped: out of time before the review was done. The work is pushed and the pull request is a draft."); + return f.done("needs_human"); + } + // The reviewer starts from nothing: a review.md left in the tree would read + // as its findings if it is stopped before writing. + await f.run("rm -f review.clean review.md"); + const review = await f.agent("adversary", { + cli: "claude", + model: "claude-sonnet-5", + task: task + "\n" + "Review the PR diff, tests, and all PR comments. Find bugs and edge cases. Write review.md. Create review.clean only if no issues remain.", + timeout: reviewLimit + "m", + }); + // A reviewer stopped at its limit did not finish, whatever it left + // behind: its review is unresolved, never clean. + const reviewTimedOut = timedOut(review); + if (reviewTimedOut) console.error("adversary was stopped at its " + reviewLimit + "m limit before it finished, so its review is not clean and is treated as unresolved."); + const clean = !reviewTimedOut && (await f.run("test -f review.clean && echo yes || echo no")).trim() === "yes"; + // Unresolved feedback stops the flow short of approval. + if (!clean) { + await f.run("review_timeout=" + (reviewTimedOut ? "yes" : "no") + "; review_limit=" + reviewLimit + "; " + reviewBlockedCommand, { timeout: "1m" }); + // report-review-findings: the run outcome says only that "its own checks + // did not pass", so this step prints why (a bounded excerpt of review.md) + // where the journal and flows status show it. Move this text into + // done("step_failed", { detail }) once AgentWorkforce/flows#542 ships. + const reviewFindings = (await f.run(reportReviewFindingsCommand)).trim(); + // Says only what is certain: the step above reports per branch whether it + // could draft the pull request or comment on it. + console.error("The adversarial review did not pass. The findings are in review-blocked.md, and on the pull request if it could be reached. This branch is not approved.\n" + reviewFindings); + return f.done("step_failed"); + } + + // Require approving reviews and passing CI checks in GitHub branch rules. + // Stop for human review. This flow never merges the pull request. + // Review and merge in GitHub; approval happens outside the runner. + return f.done("needs_human"); + +});