Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/pr-pipeline/perf-17460-defer-reviews-948b2f21.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
bump: minor
---

### Added

- Both hosted review callers' gates defer a first attempt (`defer: "true"`, `defer-job: review`) while the GITHUB_TOKEN window is projected to run short before the review ends, reading their horizon from the new `defer-run-minutes-review` (github-token default 11, the p90 of 418 hosted review runs) through `pr-pipeline.yaml`'s `limits.api` over `rate-limits.json`. Their stop notice now prints `cause=<cause> reset=<epoch>`. The lost-run reconciler reads that notice: it waits until a minute past the reset, then re-runs the deferred run once, recorded, in its own band of at most 12 re-runs a pass beside the 3 places other recovery takes; the reconciler workflow uploads up to 15 records a pass. The re-run's attempt 2 never defers ([#17460](https://github.com/melodic-software/claude-code-plugins/issues/17460)).
9 changes: 7 additions & 2 deletions .github/actions/check-kill-switch/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ to the job's horizon, or to the reset when that comes first, leaves under github
`defer-project` share of the header's limit. The horizon, the p90 of the job's measured run time,
is the gate's `defer-minutes` when set, else github-token's `defer-run-minutes-<defer-job>`, else
`defer-run-minutes`. `defer-run-minutes-pr-merge` is 6: the p90 of the 29 first-attempt pr-merge
arm runs listed on 2026-11-20 is 327 s. The github-token values are read from `resolved-config`'s `limits.api` (`pr-pipeline.yaml`) over `rate-limits.json`
arm runs listed on 2026-11-20 is 327 s. `defer-run-minutes-review` is 11, for both hosted review
callers' gates: the p90 of the 418 hosted review runs of 2 minutes or more among the 300 newest of
each caller, listed on 2026-11-21, is 10.4 minutes. The github-token values are read from `resolved-config`'s `limits.api` (`pr-pipeline.yaml`) over `rate-limits.json`
(`defer-project` 0.3, `defer-run-minutes` 15: on the mean burn curve of the 16 windows that ran
out between 2026-11-18 and 2026-11-20, at a median of minute 50, those values start deferring at
about minute 23, and the mean curve of the 17 windows that did not run out never reaches them). `defer-project` 0
Expand Down Expand Up @@ -86,7 +88,10 @@ and the pr-unblock legs (its dispatcher and pr-refine), which fix existing pull
which a person or a pull request starts, keep it too.

Set `defer: "true"` instead on a gate that finishes existing work only where a later run that never
defers recovers it: pr-merge's PR-scoped arm run (`inputs.pr`), whose work the 30-minute sweep does.
defers recovers it: pr-merge's PR-scoped arm run (`inputs.pr`), whose work the 30-minute sweep does,
and the two hosted review callers' gates, whose stop notice prints `cause=defer reset=<epoch>`, so
the lost-run reconciler re-runs the run once that reset is past (reconcile.mjs `DEFER_NOTE`, in its
own `DEFER_MAX` band).
There a low budget delays the work rather than throwing it away, which is the reason the paragraph
above keeps the budget off. CI Status and pr-dispatch-feedback never defer. A budget or defer stop
ends the run neutral: a notice, no failed step. A job that sets `budget: "true"` or `defer: "true"`
Expand Down
18 changes: 14 additions & 4 deletions .github/actions/check-kill-switch/check-kill-switch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,11 @@ test("the resolved config overrides the shipped share and horizon", async () =>
assert.equal((await runMain(DEFER, reading(2000, 1800), { resolved: config({ "defer-project": 0.25, "defer-run-minutes": 5 }) })).outputs.cause, "ok", why);
});

test("the shipped horizon for defer-job review is 11 minutes, the hosted reviews' measured p90", async () => {
const gate = await deferGate({ rateLimits: SHIPPED_LIMITS, job: "review", log: () => {} });
assert.deepEqual([gate.minutes, gate.labels.minutes], [11, "defer-run-minutes-review"]);
});

test("the shipped horizon for defer-job pr-merge is 6 minutes, and an unset job key reads defer-run-minutes", async () => {
assert.equal((await deferGate({ rateLimits: SHIPPED_LIMITS, job: "pr-merge", log: () => {} })).minutes, 6);
const other = await deferGate({ rateLimits: SHIPPED_LIMITS, job: "pr-verify", log: () => {} });
Expand Down Expand Up @@ -506,9 +511,9 @@ const GATES = {
"pr-dispatch-feedback.yml": { "post-verify-notes": "switch", dispatch: "switch", "rerun-ci-status": "switch" },
"pr-merge.yml": { "dispatch-post-merge-runs": "switch", merge: "switch", "enable-auto-merge": "defer" },
"pr-rate-merge-risk.yml": { "rate-merge-risk": "switch", "rate-merge-risk-subscription": "switch", "rate-merge-risk-fallback": "switch" },
"pr-review-hosted.yml": { "check-kill-switch": "switch" },
"pr-review-hosted.yml": { "check-kill-switch": "defer" },
"pr-review-main.yml": { "check-trigger": "switch", "collect-trusted-text": "switch", review: "switch", "review-subscription": "switch", "review-fallback": "switch" },
"pr-review-security-hosted.yml": { "check-kill-switch": "switch" },
"pr-review-security-hosted.yml": { "check-kill-switch": "defer" },
// check-broker reads the budget only for a caller passing start-gate, and
// only on a call that is the first in its slot (FIRST_CALL).
"pr-run-activity-read.yml": { "check-broker": "first-call", run: "switch" },
Expand Down Expand Up @@ -542,7 +547,11 @@ const GATES = {
"work-unblock.yml": { "label-unblocked": "start", "publish-lane-level": "switch", "sync-status": "switch" },
};

const DEFER_INPUT = /^ {10}defer: \$\{\{ inputs\.pr && 'true' \|\| 'false' \}\}$/m;
const DEFER_INPUTS = {
"pr-merge.yml": /^ {10}defer: \$\{\{ inputs\.pr && 'true' \|\| 'false' \}\}$/m,
"pr-review-hosted.yml": /^ {10}defer: "true"$/m,
"pr-review-security-hosted.yml": /^ {10}defer: "true"$/m,
};
const FIRST_CALL = /^ {10}budget: \$\{\{ inputs\.start-gate && \(inputs\.fallback == '' \|\| inputs\.fallback == 'credit-stop' \|\| inputs\.fallback == 'no-key'\) && 'true' \|\| 'false' \}\}$/m;
const FIRST_CALL_PROJECT = /^ {10}project: \$\{\{ inputs\.start-gate && \(inputs\.fallback == '' \|\| inputs\.fallback == 'credit-stop' \|\| inputs\.fallback == 'no-key'\) && 'true' \|\| 'false' \}\}$/m;

Expand Down Expand Up @@ -604,7 +613,8 @@ test("every gate in the workflows is classified, and only start gates read the b
assert.match(step, FIRST_CALL, `${file} ${job} starts new work on a starting caller's first call only`);
assert.match(step, FIRST_CALL_PROJECT, `${file} ${job} projects the window on a starting caller's first call only`);
} else if (kind === "defer") {
assert.match(step, DEFER_INPUT, `${file} ${job} defers its PR-scoped run only`);
assert.match(step, DEFER_INPUTS[file], `${file} ${job} defers only the runs a later run recovers`);
assert.match(step, /^ {10}resolved-config: \$\{\{ runner\.temp \}\}\/defer-config\.json$/m, `${file} ${job} reads the repository's override`);
assert.match(step, /^ {10}defer-job: [a-z0-9-]+$/m, `${file} ${job} names its horizon key`);
assert.doesNotMatch(step, /^ {10}defer-minutes:/m, `${file} ${job} reads its horizon from the config`);
assert.doesNotMatch(step, /budget:|project:/, `${file} ${job} reads no start-gate floor`);
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/log-api-budget/rate-guard.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ const DEFAULTS = new URL("./rate-limits.json", import.meta.url);
// (rate-guard.test.mjs holds the two equal).
const SHARES = { plan: 0.7, "writer-pause": 0.4, breaker: 0.2, "read-floor": 0.1, "stop-floor": 0.02 };
export const BUILT_IN_LIMITS = Object.freeze({
"github-token": { ...SHARES, "limit-cap": 1000, pace: 0.2, "pace-pr-merge": 0.3, "start-gate": 0.1, "start-project": 0.3, "start-run-minutes": 21, "defer-project": 0.3, "defer-run-minutes": 15, "defer-run-minutes-pr-merge": 6 },
"github-token": { ...SHARES, "limit-cap": 1000, pace: 0.2, "pace-pr-merge": 0.3, "start-gate": 0.1, "start-project": 0.3, "start-run-minutes": 21, "defer-project": 0.3, "defer-run-minutes": 15, "defer-run-minutes-pr-merge": 6, "defer-run-minutes-review": 11 },
"lanes-app": SHARES,
graphql: SHARES,
"owner-token": SHARES,
Expand All @@ -129,7 +129,7 @@ const COUNT = (value) => Number.isInteger(value) && value >= 1;
// optional, the share of the header's limit left at which the latch stops;
// start-gate, optional, check-kill-switch's share under which a lane does not
// start, which that gate reads from rate-limits.json alone.
const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "stop-floor": SHARE, "limit-cap": COUNT, pace: SHARE, "pace-pr-merge": SHARE, "start-gate": SHARE, "start-project": SHARE, "start-run-minutes": COUNT, "defer-project": SHARE, "defer-run-minutes": COUNT, "defer-run-minutes-pr-merge": COUNT };
const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "stop-floor": SHARE, "limit-cap": COUNT, pace: SHARE, "pace-pr-merge": SHARE, "start-gate": SHARE, "start-project": SHARE, "start-run-minutes": COUNT, "defer-project": SHARE, "defer-run-minutes": COUNT, "defer-run-minutes-pr-merge": COUNT, "defer-run-minutes-review": COUNT };
const CONTENT_FIELDS = { ...FIELDS, "per-minute": COUNT, "per-hour": COUNT };
// Fields rate-limits.json holds that pr-pipeline.yaml's limits.api cannot set
// (its schema rejects them): their one reader takes the file's value alone.
Expand Down
2 changes: 1 addition & 1 deletion .github/actions/log-api-budget/rate-limits.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"github-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "stop-floor": 0.02, "limit-cap": 1000, "pace": 0.2, "pace-pr-merge": 0.3, "start-gate": 0.1, "start-project": 0.3, "start-run-minutes": 21, "defer-project": 0.3, "defer-run-minutes": 15, "defer-run-minutes-pr-merge": 6 },
"github-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "stop-floor": 0.02, "limit-cap": 1000, "pace": 0.2, "pace-pr-merge": 0.3, "start-gate": 0.1, "start-project": 0.3, "start-run-minutes": 21, "defer-project": 0.3, "defer-run-minutes": 15, "defer-run-minutes-pr-merge": 6, "defer-run-minutes-review": 11 },
"lanes-app": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "stop-floor": 0.02 },
"graphql": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "stop-floor": 0.02 },
"owner-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "stop-floor": 0.02 },
Expand Down
5 changes: 3 additions & 2 deletions .github/actions/resolve-config/callers.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ for (const [file, job, lane, laneSwitch] of [
test(`${file}: the review runs only when the check-kill-switch job says proceed`, () => {
const review = jobs[job];
const gate = jobs["check-kill-switch"];
assertKillSwitchJob(gate);
assertKillSwitchJob(gate, { defers: true });
for (const proceed of ["true", "false", ""]) {
const context = {
"github.event.pull_request.draft": false,
Expand Down Expand Up @@ -272,7 +272,7 @@ function importClosure(entry) {
// The check-kill-switch job reads the action from the default branch with a
// read-only token and passes it the org and repository switches; a gate
// that starts new work also passes budget "true".
function assertKillSwitchJob(gate, { starts = false } = {}) {
function assertKillSwitchJob(gate, { starts = false, defers = false } = {}) {
assert.deepEqual(gate.permissions, { contents: "read" });
const checkout = gate.steps.find((step) => step.uses?.startsWith("actions/checkout@"));
assertMainTip(gate.steps, checkout);
Expand All @@ -285,6 +285,7 @@ function assertKillSwitchJob(gate, { starts = false } = {}) {
value: "${{ vars.CLAUDE_LANES_DISABLED }}",
"repo-value": "${{ vars.CLAUDE_LANES_REPO_DISABLED }}",
...(starts && { budget: "true" }),
...(defers && { defer: "true", "defer-job": "review", "resolved-config": "${{ runner.temp }}/defer-config.json" }),
});
}

Expand Down
64 changes: 55 additions & 9 deletions .github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,12 @@
// API rate limit (`rateHeld`), named by ci-status's RATE_HELD_NOTE, goes the
// same way, once RATE_RESET_GRACE_MS past the latest reset its refusals
// name; the merge gate binds that attempt 2 (babysit_review_evidence
// `_rate_held`).
// `_rate_held`). A first attempt the defer gate stopped (its gate job's
// DEFER_NOTE, cause=defer with the window's reset) waits until
// RATE_RESET_GRACE_MS past that reset, so the first pass after it, a few
// minutes past the reset when the hourly tick follows it, re-runs it. Those
// re-runs take the DEFER_MAX band, not the places below; their attempt 2
// never defers.
// Other lost lane runs (a review lane run lost any other way, refine) are listed
// in the summary for a person: a review re-run would trade one review for
// another, and refine's lanes token broker refuses a re-run
Expand Down Expand Up @@ -228,7 +233,8 @@
// SETTLED_PATH plan reads the last pass's settled runs and job reads, then writes
// this pass's there for the upload; unset carries nothing
// GITHUB_RUN_ID, RUNNER_TEMP this run, and where the last carry is downloaded
// GITHUB_OUTPUT plan writes record-1 to record-3 there (empty when unused)
// GITHUB_OUTPUT plan writes record-1 to record-15 there (empty when unused;
// https://github.com/melodic-software/claude-code-plugins/blob/main/.github/workflows/maintenance-reconcile-lost-runs.yml)
// GITHUB_STEP_SUMMARY the outcome is listed there when set
// GITHUB_API_URL
import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
Expand Down Expand Up @@ -420,6 +426,12 @@ export const FAULT_HELD_HOURS = 6;
// for the next pass, so a pass after a long outage adds a bounded load.
export const FAULT_HELD_READS = 100;
export const MAX_IN_FLIGHT = 3;
// Re-runs of reviews the defer gate stopped, per pass, beside MAX_IN_FLIGHT:
// the 24 h to 2026-11-21T20:49Z would have deferred at most 9 review runs in
// one token window, so 12 leaves room for one more head, both lanes
// (https://github.com/melodic-software/claude-code-plugins/issues/17460).
export const DEFER_MAX = 12;
export const RECORD_SLOTS = MAX_IN_FLIGHT + DEFER_MAX;
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved
export const WRITE_GAP_MS = 1000;
// Pages of 100 read per status query of a lane's run listing; a listing that
// does not fit is a failure, never a partial read.
Expand Down Expand Up @@ -1050,6 +1062,15 @@ export const rateHeld = async (jobs, notesOf) => {
return undefined;
};
export const RATE_RESET_GRACE_MS = 60_000;
// The hosted callers' stop notice when the gate deferred the review, with the
// window's reset in epoch seconds (https://github.com/melodic-software/claude-code-plugins/blob/main/.github/actions/check-kill-switch/README.md).
export const DEFER_NOTE = /\bcause=defer reset=([0-9]{9,12})\b/;
const DEFERRED = "success (deferred)";
export const deferReset = async (jobs, notesOf) => {
const gate = jobs.find((job) => GATE_JOBS.has(job?.name) && Number.isSafeInteger(job.id) && job.steps?.some((step) => step?.name === REVIEW_STOP_STEP && step.conclusion === "success"));
const resets = gate ? (await notesOf(gate)).map((a) => DEFER_NOTE.exec(String(a?.message ?? ""))?.[1]).filter(Boolean).map(Number) : [];
return resets.length > 0 ? Math.max(...resets) : undefined;
};

/**
* Why the run's first attempt must not be re-run, with whether that hold is
Expand Down Expand Up @@ -1499,8 +1520,14 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
}
const jobs = await readJobs(r);
const stopped = reviewStopped(jobs);
const notesOf = (job) => job.annotations ?? call("GET", `/repos/${repo}/check-runs/${job.id}/annotations?per_page=100`);
const defer = stopped ? await deferReset(jobs, notesOf) : undefined;
if (beforeReset(defer === undefined ? undefined : defer * 1000, now)) {
waitReset(`#${pull.number}: review run ${r.id} (${r.path})`, defer * 1000, "was deferred on the GITHUB_TOKEN budget");
continue;
}
if (!stopped) {
const reset = await rateHeld(jobs, (job) => job.annotations ?? call("GET", `/repos/${repo}/check-runs/${job.id}/annotations?per_page=100`));
const reset = await rateHeld(jobs, notesOf);
if (reset === undefined) {
continue;
}
Expand All @@ -1511,10 +1538,10 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
}
const hold = await gateHold();
if (hold) {
say(`#${pull.number}: review run ${r.id} (${r.path}) ${stopped ? "was stopped by a kill switch" : "held on a GitHub API rate limit"}; waiting, ${hold}`);
say(`#${pull.number}: review run ${r.id} (${r.path}) ${defer !== undefined ? "was deferred on the GITHUB_TOKEN budget" : stopped ? "was stopped by a kill switch" : "held on a GitHub API rate limit"}; waiting, ${hold}`);
continue;
}
const conclusion = stopped ? "success (kill switch stop)" : "success (rate-limit hold)";
const conclusion = defer !== undefined ? DEFERRED : stopped ? "success (kill switch stop)" : "success (rate-limit hold)";
items.push({ kind: "review", pr: pull.number, id: r.id, path: r.path, conclusion, attempt: 1, head: pull.head.sha, record: rerunRecordName(r.id, 2, pull.head.sha) });
}
return items;
Expand Down Expand Up @@ -1966,7 +1993,26 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
: places(set) > room
? `only ${room} of ${MAX_IN_FLIGHT} re-run places left`
: "";
const place = (what, set) => {
let deferRoom = DEFER_MAX;
const takeDeferred = (what, set) => {
const reviews = set.filter(isReview);
if (reviews.length === 0 || !reviews.every((item) => item.conclusion === DEFERRED)) {
return set;
}
if (reviews.length > deferRoom) {
deferred += 1;
say(`${what}: its deferred reviews wait, only ${deferRoom} of ${DEFER_MAX} recovery places left this pass`);
} else {
items.push(...reviews);
deferRoom -= reviews.length;
}
return set.filter((item) => !isReview(item));
};
const place = (what, whole) => {
const set = takeDeferred(what, whole);
if (set.length === 0) {
return;
}
const core = set.filter((item) => !isReview(item));
const split = short(set) === NO_UPLOAD && places(set) <= room && core.length > 0 && core.length < set.length && short(core) === "";
if (split) {
Expand Down Expand Up @@ -2062,7 +2108,7 @@ async function act({ repo, call, say, since, sleep, items }) {
: item.kind === "too-old"
? `${where}: re-ran run ${item.id} (${item.path}) so pr-verify judges the head again: the pull request is past ${TOO_OLD_DAYS} days and its head has no verdict`
: item.kind === "review"
? `${where}: re-ran review run ${item.id} (${item.path}), which ${item.conclusion === "success (rate-limit hold)" ? "held on a GitHub API rate limit" : "a kill switch stopped"}; its completion re-runs ci-status`
? `${where}: re-ran review run ${item.id} (${item.path}), which ${item.conclusion === "success (rate-limit hold)" ? "held on a GitHub API rate limit" : item.conclusion === DEFERRED ? "the defer gate stopped until the GITHUB_TOKEN window reset" : "a kill switch stopped"}; its completion re-runs ci-status`
: item.kind === "fault-held"
? `${where}: re-ran run ${item.id} (${item.path}), which ${item.path === RATER ? "the lanes kill switch stopped before it rated" : "held on a GitHub fault"}`
: `${where}: re-ran run ${item.id} (${item.path}), which ended ${item.conclusion}`,
Expand All @@ -2083,10 +2129,10 @@ async function act({ repo, call, say, since, sleep, items }) {
// The plan's record-N outputs, each a record name or empty.
function recordOutputs(items) {
const names = items.filter(recorded).map((item) => item.record);
if (names.length > MAX_IN_FLIGHT || !names.every(isRecordName)) {
if (names.length > RECORD_SLOTS || !names.every(isRecordName)) {
throw new Error("the plan's record names are malformed");
}
return Object.fromEntries(Array.from({ length: MAX_IN_FLIGHT }, (_, i) => [`record-${i + 1}`, names[i] ?? ""]));
return Object.fromEntries(Array.from({ length: RECORD_SLOTS }, (_, i) => [`record-${i + 1}`, names[i] ?? ""]));
}

export async function main({ env = process.env, github, fetchStatus: readStatus = fetchStatus, sleep = delay, now = Date.now(), download } = {}) {
Expand Down
Loading
Loading