Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/pr-pipeline/main-371a8c8b.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
bump: none
---

Moves the run-claude-cli link in pr-run-activity.md to ci-workflows v0.76.3; the contract does not change.
3 changes: 0 additions & 3 deletions .github/actions/log-api-budget/rate-guard.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -888,9 +888,6 @@ const UNGUARDED = {
".github/scripts/pr-unblock/address-feedback.mjs": "imported by script activities run from a directory copy",
// pr-merge's own guard is its owner's change.
".github/scripts/pr-merge/merge-holds.mjs": "pr-merge, guarded by its owner",
// A test helper no lane runs: the credential-legs tests' read of ci-workflows
// files at a pinned SHA from raw.githubusercontent.com, not the API.
".github/scripts/lib/model-credential.mjs": "test helper reading raw.githubusercontent.com, not the API",
// Its API calls go through lib/github-rest.mjs, which imports the guard; the
// one raw fetch is getStatus, reading www.githubstatus.com, not the API.
".github/scripts/maintenance-monitor-signals/monitor.mjs": "githubstatus.com reads; API calls via lib/github-rest.mjs",
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/resolve-config/callers.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { retryFetch } from "../../scripts/lib/retry-fetch.mjs";
import { DEFAULTS as CI_FIX_DEFAULTS, skipReason } from "../../scripts/pr-unblock/fix-ci-dispatch.mjs";
import { VERDICTS } from "../../scripts/pr-unblock/review-dependency.mjs";
import { stopSkip } from "../../scripts/pr-unblock/escalation-routes.mjs";
import { laneAtPin } from "../../scripts/lib/ci-workflows-at-pin.mjs";
import { contractAtPin } from "../../scripts/lib/ci-workflows-at-pin.mjs";
import { levelSettings } from "../../scripts/lib/lane-level-value.mjs";
import { main as checkTrustedTrigger } from "../check-trusted-trigger/check-trusted-trigger.mjs";
import { generate as generateCarrierKey } from "../open-lane-output/keygen.mjs";
Expand Down Expand Up @@ -2796,7 +2796,7 @@ test("every run-claude-cli step pin in this repo's workflows passes --permission
for (const file of ["pr-run-activity-read.yml", "pr-verify.yml", "pr-unblock.yml"]) assert.ok(files.has(file), [...files].join(", "));
assert.ok(modes.size > 0, "a launcher step names its permission-mode");
for (const pin of pins) {
const { cliArgs } = await laneAtPin(pin);
const { cliArgs } = await contractAtPin(pin);
for (const permissionMode of new Set([...modes, "default"]))
for (const sandbox of [true, false])
for (const effort of ["", "high"])
Expand Down
12 changes: 8 additions & 4 deletions .github/scripts/lib/ci-workflows-at-pin.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";

import { CLASSIFIER_FILES, loadLauncher } from "../plugin-eval/eval-handoff.mjs";
import { CONTRACT_FILES, contractFiles, loadContract } from "../plugin-eval/eval-handoff.mjs";

const fetched = new Map();
const ATTEMPTS = 3;
Expand All @@ -26,14 +26,18 @@ export function fetchAtPin(pin, path) {
return fetched.get(url);
}

export async function laneAtPin(pin) {
export async function contractFilesAtPin(pin) {
return contractFiles(await fetchAtPin(pin, CONTRACT_FILES));
}

export async function contractAtPin(pin) {
const root = mkdtempSync(join(tmpdir(), "ci-workflows-at-pin-"));
try {
for (const path of CLASSIFIER_FILES) {
for (const path of [CONTRACT_FILES, ...(await contractFilesAtPin(pin))]) {
mkdirSync(dirname(join(root, path)), { recursive: true });
writeFileSync(join(root, path), await fetchAtPin(pin, path));
}
return loadLauncher(root);
return loadContract(root);
} finally {
rmSync(root, { recursive: true, force: true });
}
Expand Down
36 changes: 4 additions & 32 deletions .github/scripts/lib/model-credential.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,8 @@
// semantics, so a job's `if:` and an input's `${{ }}` evaluate as GitHub
// evaluates them.
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { dirname, join, posix } from "node:path";

import { contractAtPin } from "./ci-workflows-at-pin.mjs";

// The one pin every select-model-credential step in the given workflows names.
export function selectPin(...workflowTexts) {
Expand All @@ -17,35 +15,9 @@ export function selectPin(...workflowTexts) {
return [...pins][0];
}

// Fetches `entry` and every relative file it requires, at `pin`, into a
// fresh directory, and requires it from there.
async function fetchModule(pin, entry) {
const root = mkdtempSync(join(tmpdir(), "ci-workflows-"));
const seen = new Set();
const queue = [entry];
while (queue.length) {
const path = queue.shift();
if (seen.has(path)) continue;
seen.add(path);
const url = `https://raw.githubusercontent.com/melodic-software/ci-workflows/${pin}/${path}`;
const response = await fetch(url, { signal: AbortSignal.timeout(15000) });
assert.ok(response.ok, `${url} answered ${response.status}`);
const text = await response.text();
mkdirSync(join(root, dirname(path)), { recursive: true });
writeFileSync(join(root, path), text);
for (const [, spec] of text.matchAll(/require\(\s*"(\.{1,2}\/[^"]+)"\s*\)/g)) {
queue.push(posix.normalize(posix.join(posix.dirname(path), spec)));
}
}
return createRequire(join(root, "package.json"))(`./${entry}`);
}

// The action's start/leg/next and the launcher's handoffReason, at `pin`.
export async function loadSelect(pin) {
const select = await fetchModule(pin, ".github/actions/select-model-credential/select.cjs");
const { handoffReason } = await fetchModule(pin, ".github/actions/run-claude-cli/lane.cjs");
assert.equal(typeof handoffReason, "function", "lane.cjs exports handoffReason at the pin");
return { ...select, handoffReason };
const { HANDOFF_REASONS, handoffReason, leg, next, start } = await contractAtPin(pin);
return { HANDOFF_REASONS, handoffReason, leg, next, start };
}

// One expression, `${{ }}` or bare, over `context` (a flat map of
Expand Down
8 changes: 4 additions & 4 deletions .github/scripts/plugin-eval/credential-legs.test.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
// maintenance-measure-skill-edit-pass.yml and pr-run-plugin-evals.yml pick each leg's
// credential through ci-workflows' select-model-credential, fetched at the pin
// the workflows name. A leg's handoff comes from eval-handoff.mjs (its
// handoffReason, the rule run-claude-cli's lane.cjs applies), so this truth
// the workflows name. A leg's handoff comes from eval-handoff.mjs (the
// handoffReason run-claude-cli's contract exports:
// https://github.com/melodic-software/ci-workflows/blob/main/.github/actions/run-claude-cli/contract.cjs), so this truth
// table drives every handoff that rule can give, with the FALLBACK_READY each
// leg passes it, through each workflow's own conditions and inputs, over every
// combination of gate verdict, gate job result, secrets set or empty, and a
Expand All @@ -16,7 +17,6 @@ import { test } from "node:test";

import { loadSelect, runs, selectPin, value } from "../lib/model-credential.mjs";
import { workflowText } from "../lib/workflow-text.mjs";
import { handoffReason } from "./eval-handoff.mjs";

const read = (name) => readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8");
const HILL = read("maintenance-measure-skill-edit-pass.yml");
Expand Down Expand Up @@ -111,7 +111,7 @@ function facts(leg) {
}
return [...seen.values()];
}
const reasonOf = (leg, f, fallbackReady) => handoffReason(EVAL_LEG[leg], f.failureClass, f.status, { fallbackReady, limitStop: f.limitStop, creditStop: f.creditStop });
const reasonOf = (leg, f, fallbackReady) => S.handoffReason({ leg: EVAL_LEG[leg] }, f.failureClass, f.status, { fallbackReady, limitStop: f.limitStop, creditStop: f.creditStop });

// The judge step's outputs: it runs only when the leg's credential let the
// probe run and no step before the probe failed; its handoff is
Expand Down
63 changes: 23 additions & 40 deletions .github/scripts/plugin-eval/eval-handoff.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
// as stream-json, one message per line (code.claude.com/docs/en/plugin-evals,
// "What a grader can look at", the `trace` row), the same stream run-claude-cli
// writes as its execution file. So every errored run's trace and the probe's
// stream go through lane.cjs judgeLeg and classify.cjs isCreditStop from the
// ci-workflows checkout at the pin the lanes run (CI_WORKFLOWS), and the leg
// hands off on the rule lane.cjs handoffReason applies, restated below because
// lane.cjs does not export it:
// stream go through judgeLeg and isCreditStop from run-claude-cli's published
// contract in the ci-workflows checkout at the pin the lanes run (CI_WORKFLOWS),
// https://github.com/melodic-software/ci-workflows/blob/main/.github/actions/run-claude-cli/contract.cjs,
// and the leg hands off on the contract's handoffReason:
// key leg a credit stop, or class auth with API status 401 or 402
// subscription leg with FALLBACK_READY true: a credit stop, a 401 or 402,
// or a limit stop (a rejected rate_limit_event with its reset)
Expand Down Expand Up @@ -39,22 +39,10 @@ import process from "node:process";
import { fileURLToPath } from "node:url";

const LEGS = { key: "api-key", subscription: "subscription", fallback: "fallback" };
const REFUSED_STATUSES = [401, 402];
const AUTH_FAILED_STATUS = 401;
// The classes a run's trace can show that say the credential stopped, not the case.
const STOP_CLASSES = ["credit-stop", "auth", "rate-limit", "overloaded"];

export function handoffReason(leg, failureClass, status, { fallbackReady = false, limitStop = false, creditStop = false } = {}) {
const refused = failureClass === "auth" && REFUSED_STATUSES.includes(status);
if (leg === "api-key") {
if (failureClass === "credit-stop") return "credit-stop";
return refused ? "auth" : "";
}
if (leg !== "subscription" || !fallbackReady || failureClass === "") return "";
if (creditStop) return "credit-stop";
if (refused) return "auth";
return failureClass === "rate-limit" && limitStop ? "limit-stop" : "";
}

// Every errored run's trace path, resolved as run-validity.py resolves it.
export function erroredTraces(result, base) {
const traces = [];
Expand All @@ -71,14 +59,14 @@ export function erroredTraces(result, base) {
}

// One stream's verdict: its class and why it hands off ("" when it does not).
export function judgeStream(text, exitCode, { leg, fallbackReady }, { judgeLeg, linesToMessages, isCreditStop }) {
export function judgeStream(text, exitCode, { leg, fallbackReady }, { judgeLeg, linesToMessages, isCreditStop, handoffReason }) {
const dir = mkdtempSync(join(tmpdir(), "eval-handoff-"));
try {
const executionFile = join(dir, "execution.json");
writeFileSync(executionFile, JSON.stringify(linesToMessages(text)));
const judged = judgeLeg({ leg: { leg }, executionFile, exitCode, timedOut: false, maxTurns: "" });
const status = judged.messages.at(-1)?.api_error_status;
const reason = handoffReason(leg, judged.failureClass, Number.isInteger(status) ? status : null, {
const reason = handoffReason({ leg }, judged.failureClass, Number.isInteger(status) ? status : null, {
fallbackReady,
limitStop: judged.failureClass === "rate-limit" && judged.resets !== null,
creditStop: leg === "subscription" && isCreditStop(judged.messages, { exitCode }),
Expand Down Expand Up @@ -108,9 +96,9 @@ function authFailed(result) {

// The streams' verdict, overruled by a suite that ended auth_failed unless a
// stream already handed off.
export function withPartials(verdict, results, { leg, fallbackReady }) {
export function withPartials(verdict, results, { leg, fallbackReady }, { handoffReason }) {
if (verdict.handoff || !results.some(authFailed)) return verdict;
const reason = handoffReason(leg, "auth", REFUSED_STATUSES[0], { fallbackReady });
const reason = handoffReason({ leg }, "auth", AUTH_FAILED_STATUS, { fallbackReady });
return { handoff: reason !== "", reason, laneClass: "auth" };
}

Expand All @@ -122,28 +110,23 @@ function readJson(path) {
}
}

// The classifier's four files (lane.cjs and the three it requires), copied
// under a folder of their own whose package.json ends Node's package-scope
// lookup there. Required in place, the lookup climbs past the sparse checkout
// to the workspace root, and a package.json there that does not parse stops
// the require (run 42138674762).
export const CLASSIFIER_FILES = [
".github/actions/run-claude-cli/lane.cjs",
".github/actions/run-claude-cli/managed-settings.cjs",
".github/actions/report-lane-outcome/classify.cjs",
".github/scripts/resolve-npm-tool-version.cjs",
];
export function loadLauncher(root) {
const dir = mkdtempSync(join(tmpdir(), "eval-handoff-classifier-"));
export const CONTRACT = ".github/actions/run-claude-cli/contract.cjs";
export const CONTRACT_FILES = ".github/actions/run-claude-cli/contract-files.txt";
export const contractFiles = (manifest) => manifest.split("\n").map((line) => line.trim()).filter(Boolean);

// Copied under a folder whose package.json ends Node's package-scope lookup
// there, as contract.cjs asks: required in place, the lookup climbs to the
// workspace root, whose package.json may not parse (run 42138674762):
// https://github.com/melodic-software/ci-workflows/blob/main/.github/actions/run-claude-cli/contract.cjs
export function loadContract(root) {
const dir = mkdtempSync(join(tmpdir(), "eval-handoff-contract-"));
try {
writeFileSync(join(dir, "package.json"), '{"type":"commonjs"}\n');
for (const path of CLASSIFIER_FILES) {
for (const path of contractFiles(readFileSync(join(root, CONTRACT_FILES), "utf8"))) {
mkdirSync(dirname(join(dir, path)), { recursive: true });
copyFileSync(join(root, path), join(dir, path));
}
const require = createRequire(join(dir, CLASSIFIER_FILES[0]));
const { isCreditStop } = require("../report-lane-outcome/classify.cjs");
return { ...require("./lane.cjs"), isCreditStop };
return createRequire(join(dir, "package.json"))(`./${CONTRACT}`);
} finally {
rmSync(dir, { recursive: true, force: true });
}
Expand All @@ -152,7 +135,7 @@ export function loadLauncher(root) {
function main(env) {
const leg = LEGS[env.LEG];
if (!leg) throw new Error(`LEG must be key, subscription or fallback, not ${JSON.stringify(env.LEG)}`);
const launcher = loadLauncher(env.CI_WORKFLOWS ?? "");
const launcher = loadContract(env.CI_WORKFLOWS ?? "");
const streams = [];
if (env.PROBE && existsSync(env.PROBE)) {
const exitCode = Number.parseInt(env.PROBE_EXIT ?? "", 10);
Expand All @@ -172,7 +155,7 @@ function main(env) {
}
}
const options = { leg, fallbackReady: env.FALLBACK_READY === "true" };
const verdict = withPartials(judgeLegStreams(streams, options, launcher), results, options);
const verdict = withPartials(judgeLegStreams(streams, options, launcher), results, options, launcher);
if (env.LEG_RESULT) {
const message = {
type: "result",
Expand Down
Loading
Loading