Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/actions/resolve-config/callers.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9126,6 +9126,26 @@ test("every model leg is followed by a sealed transcript upload of its own execu
assert.ok(legs >= 13, `only ${legs} model legs found`);
});

test("a transcript upload's otel-file is its own leg's, from a launcher with the receiver on", () => {
let carried = 0;
for (const file of workflowFiles()) {
for (const [jobId, job] of Object.entries(workflow(file).jobs ?? {})) {
const steps = job.steps ?? [];
for (const [at, step] of steps.entries()) {
if (step.with?.["otel-receiver"] !== undefined) assert.ok(isLauncher(step), `${file}#${jobId} ${step.name}: otel-receiver on a step that is no launcher`);
if (!isTranscriptUpload(step) || step.with?.["otel-file"] === undefined) continue;
carried++;
const leg = /^\$\{\{ steps\.([\w-]+)\.outputs\.execution-file \}\}$/.exec(step.with["execution-file"] ?? "")?.[1];
assert.equal(step.with["otel-file"], `\${{ steps.${leg}.outputs.otel-file }}`, `${file}#${jobId} ${step.name}: the otel-file of the leg whose transcript it uploads`);
const launcher = steps.slice(0, at).find((s) => s.id === leg);
assert.ok(launcher && isLauncher(launcher), `${file}#${jobId} ${step.name}: ${leg} is a launcher before it`);
assert.equal(launcher.with?.["otel-receiver"], "true", `${file}#${jobId} ${leg}: the receiver is on`);
}
}
}
if (workflowFiles().includes("maintenance-audit-lane-runs.yml")) assert.ok(carried >= 3, `only ${carried} uploads carry telemetry`);
});

test("every caller of a reusable holding model legs passes the transcript key", () => {
let callers = 0;
for (const file of workflowFiles()) {
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/scan-upload/scan-upload.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,11 @@ export function main(env = process.env) {
if (withheld) {
if (withheld.fingerprint && env.GITHUB_STEP_SUMMARY) {
try {
appendFileSync(env.GITHUB_STEP_SUMMARY, `### scan-upload withheld the upload\n\n${withheld.fingerprint}\n`);
appendFileSync(env.GITHUB_STEP_SUMMARY, `### scan-upload ${env.SCAN_WITHHOLDS ? `hit: ${env.SCAN_WITHHOLDS}` : "withheld the upload"}\n\n${withheld.fingerprint}\n`);
} catch {}
}
process.stdout.write(
`::error::scan-upload: ${withheld.label}; nothing is uploaded${withheld.credential ? ". Rotate this credential" : ""}\n`,
`::error::scan-upload: ${withheld.label}; ${env.SCAN_WITHHOLDS || "nothing is uploaded"}${withheld.credential ? ". Rotate this credential" : ""}\n`,
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved
);
return 1;
}
Expand Down
19 changes: 17 additions & 2 deletions .github/actions/scan-upload/scan-upload.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ function workDir(t) {
return dir;
}

function run(dir, paths) {
function run(dir, paths, extra = {}) {
const output = path.join(dir, "github-output");
const summary = path.join(dir, "step-summary");
writeFileSync(output, "");
writeFileSync(summary, "");
const result = spawnSync(process.execPath, [SCRIPT], {
env: { PATH: process.env.PATH, SCAN_PATHS: paths, GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary },
env: { PATH: process.env.PATH, SCAN_PATHS: paths, GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, ...extra },
encoding: "utf8",
});
return { ...result, outputs: readFileSync(output, "utf8"), summary: readFileSync(summary, "utf8") };
Expand Down Expand Up @@ -64,6 +64,16 @@ test("a planted credential withholds the upload and asks for rotation, never pri
assert.match(result.stdout, /Rotate this credential/);
});

test("SCAN_WITHHOLDS names what a hit withholds in place of the whole upload", (t) => {
const dir = workDir(t);
writeFileSync(path.join(dir, "otel.jsonl"), `{"resourceLogs":[{"body":"${PLANTED}"}]}\n`);
const result = run(dir, path.join(dir, "otel.jsonl"), { SCAN_WITHHOLDS: "the telemetry is left out and the transcript still uploads" });
withheld(result, "GitHub token; the telemetry is left out and the transcript still uploads");
assert.doesNotMatch(result.stdout, /nothing is uploaded/);
assert.match(result.summary, /^### scan-upload hit: the telemetry is left out and the transcript still uploads\n/);
assert.doesNotMatch(result.summary, /withheld the upload/);
});

test("a block writes only the hit's rule, path and line to the summary, no character of the match", (t) => {
const dir = workDir(t);
mkdirSync(path.join(dir, "out", "nested"), { recursive: true });
Expand Down Expand Up @@ -166,4 +176,9 @@ test("the lane transcript is scanned as plaintext before it is sealed", () => {
assert.ok(scan > 0 && scan < seal, "the scan step comes before the seal step");
assert.match(action, /SCAN_PATHS: \$\{\{ inputs\.execution-file \}\}\n\s+run: node "\$GITHUB_ACTION_PATH\/\.\.\/scan-upload\/scan-upload\.mjs"/);
assert.match(action, /id: seal\n\s+if: steps\.scan\.outputs\.clean == 'true'\n/);
const scanOtel = action.indexOf("id: scan-otel\n");
const sealOtel = action.indexOf("id: seal-otel\n");
assert.ok(seal < scanOtel && scanOtel < sealOtel, "the telemetry is scanned after the transcript is sealed and before its own seal");
assert.match(action, /SCAN_PATHS: \$\{\{ inputs\.otel-file \}\}\n.*\n\s+run: node "\$GITHUB_ACTION_PATH\/\.\.\/scan-upload\/scan-upload\.mjs"/);
assert.match(action, /id: seal-otel\n\s+if: steps\.scan-otel\.outputs\.clean == 'true'\n/);
});
6 changes: 6 additions & 0 deletions .github/actions/upload-lane-transcript/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ This repository is public, so its artifacts are too; only ciphertext is uploaded
| `execution-file` | The leg's execution file: run-claude-cli's `execution-file` or claude-code-action's `execution_file` output. Empty, missing, empty-file or a symbolic link uploads nothing |
| `public-key` | `${{ vars.LANE_TRANSCRIPT_PUBLIC_KEY }}`, one line `x25519 <key id> <base64 raw key>`. Empty skips with a notice; anything else that is not a key line fails the step |
| `job-id` | A fixed key for the leg, unique in the run, matching `^[A-Za-z_][A-Za-z0-9_-]{0,99}$` |
| `otel-file` | Optional: run-claude-cli's `otel-file` output, from a leg run with `otel-receiver: "true"`. Empty uploads the transcript alone |

Callers run it with `if: always()` and `continue-on-error: true` after the leg, so a failed leg's
transcript is kept and an upload failure never changes the lane's outcome.
Expand All @@ -22,6 +23,11 @@ transcript is kept and an upload failure never changes the lane's outcome.
record for the payload name `transcript.json.gz`, bound to `GITHUB_RUN_ID` and `job-id`; its
plaintext is the execution file, gzipped.

With `otel-file`, the artifact also holds `otel.jsonl.gz.sealed`: the leg's OTLP/JSON traces and
logs, scanned on their own after the transcript is sealed, then sealed the same way under the payload
name `otel.jsonl.gz`. Telemetry that fails its scan, cannot be sealed, or would take the artifact past
the puller's 64 MiB limit is left out with an annotation; the transcript still uploads.

The [transcript puller](../../../scripts/lane-transcripts/README.md) opens it: it takes the run id
from the artifact's workflow run and the job id from the artifact name (everything between
`lane-transcript-` and the last `-`), opens the record with `openRecord` from
Expand Down
35 changes: 33 additions & 2 deletions .github/actions/upload-lane-transcript/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ description: >
transcript puller. An empty public key skips with a notice; a missing or
empty execution file uploads nothing; a malformed key or job-id fails the
step. The plaintext is scanned first (../scan-upload), and a credential in
it fails the step with nothing sealed. Needs node 18 or later on PATH,
it fails the step with nothing sealed. With otel-file, the leg's telemetry
travels in the same artifact as otel.jsonl.gz.sealed, scanned and sealed the
same way; telemetry that fails its scan, cannot be sealed, or would take the
artifact past the puller's size limit is left out with an annotation, and
the transcript still uploads. Needs node 18 or later on PATH,
../seal-lane-output and ../scan-upload beside it, and lib/publish-gate.mjs.

inputs:
Expand All @@ -19,6 +23,10 @@ inputs:
job-id:
description: A fixed key for the leg, unique in the run, that the puller reads back from the artifact name.
required: true
otel-file:
description: The leg's telemetry, run-claude-cli's otel-file output; empty (the default) uploads the transcript alone.
required: false
default: ""

runs:
using: composite
Expand All @@ -42,12 +50,35 @@ runs:
JOB_ID: ${{ inputs.job-id }}
run: node "$GITHUB_ACTION_PATH/seal-transcript.mjs"

- name: Scan the telemetry for credentials
id: scan-otel
if: inputs.otel-file != '' && steps.seal.outputs.file != ''
continue-on-error: true
shell: bash
env:
SCAN_PATHS: ${{ inputs.otel-file }}
SCAN_WITHHOLDS: the telemetry is left out and the transcript still uploads
run: node "$GITHUB_ACTION_PATH/../scan-upload/scan-upload.mjs"

- name: Seal the telemetry
id: seal-otel
if: steps.scan-otel.outputs.clean == 'true'
continue-on-error: true
shell: bash
env:
OTEL_FILE: ${{ inputs.otel-file }}
PUBLIC_KEY: ${{ inputs.public-key }}
JOB_ID: ${{ inputs.job-id }}
run: node "$GITHUB_ACTION_PATH/seal-transcript.mjs" telemetry

- name: Upload the sealed transcript
if: steps.seal.outputs.file != ''
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: lane-transcript-${{ inputs.job-id }}-${{ github.run_attempt }}
path: ${{ steps.seal.outputs.file }}
path: |
${{ steps.seal.outputs.file }}
${{ steps.seal-otel.outputs.file }}
retention-days: 7
# Ciphertext does not compress.
compression-level: 0
Expand Down
72 changes: 57 additions & 15 deletions .github/actions/upload-lane-transcript/seal-transcript.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// seal-lane-output record, written under RUNNER_TEMP for the upload step. The
// plaintext stays in memory: only the sealed record reaches disk, and nothing
// here prints it. README.md states the format.
import { appendFileSync, lstatSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { appendFileSync, lstatSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
Expand All @@ -11,6 +11,24 @@ import { gzipSync } from "node:zlib";
import { JOB_ID, SealError, parsePublicKey, sealRecord } from "../seal-lane-output/seal.mjs";

export const PAYLOAD = "transcript.json.gz";
export const TELEMETRY_PAYLOAD = "otel.jsonl.gz";
export const MAX_ARTIFACT_BYTES = 63 * 1024 * 1024;

/** True for a regular, non-empty file; a link or a missing path is not one. */
function hasContent(file) {
try {
const stat = file === "" ? undefined : lstatSync(file);
return stat?.isFile() === true && stat.size > 0;
} catch {
return false;
}
}

function jobIdOf(env) {
const jobId = env.JOB_ID ?? "";
if (!JOB_ID.test(jobId)) throw new SealError("the job-id input is not a job key");
return jobId;
}

export function main(env = process.env) {
if (String(env.PUBLIC_KEY ?? "").trim() === "") {
Expand All @@ -19,17 +37,10 @@ export function main(env = process.env) {
}
// The key first: a malformed key fails whether or not a leg ran.
const recipient = parsePublicKey(env.PUBLIC_KEY);
const jobId = env.JOB_ID ?? "";
if (!JOB_ID.test(jobId)) throw new SealError("the job-id input is not a job key");
const jobId = jobIdOf(env);
const file = env.EXECUTION_FILE ?? "";
let stat;
try {
stat = file === "" ? undefined : lstatSync(file);
} catch {
stat = undefined;
}
// No regular, non-empty file: no leg ran, so there is nothing to keep.
if (!stat?.isFile() || stat.size === 0) return;
if (!hasContent(file)) return;
const record = sealRecord(recipient, PAYLOAD, gzipSync(readFileSync(file)), env.GITHUB_RUN_ID, jobId);
const dir = path.join(env.RUNNER_TEMP, "lane-transcript", jobId);
rmSync(dir, { recursive: true, force: true });
Expand All @@ -39,11 +50,42 @@ export function main(env = process.env) {
appendFileSync(env.GITHUB_OUTPUT, `file=${sealed}\n`);
}

/**
* Seal the leg's telemetry beside its sealed transcript, under its own payload
* name and the same run and job binding. Runs after the transcript is on disk
* and never touches it; an artifact that would pass the puller's limit leaves
* the telemetry out.
*/
export function sealTelemetry(env = process.env, maxArtifactBytes = MAX_ARTIFACT_BYTES) {
const file = env.OTEL_FILE ?? "";
if (!hasContent(file)) return;
const recipient = parsePublicKey(env.PUBLIC_KEY);
const jobId = jobIdOf(env);
const dir = path.join(env.RUNNER_TEMP, "lane-transcript", jobId);
const transcriptBytes = statSync(path.join(dir, `${PAYLOAD}.sealed`)).size;
const record = sealRecord(recipient, TELEMETRY_PAYLOAD, gzipSync(readFileSync(file)), env.GITHUB_RUN_ID, jobId);
if (transcriptBytes + record.length > maxArtifactBytes) {
process.stdout.write("::warning::upload-lane-transcript: the telemetry is left out: with it the artifact would pass the transcript puller's size limit\n");
return;
}
const sealed = path.join(dir, `${TELEMETRY_PAYLOAD}.sealed`);
writeFileSync(sealed, record, { flag: "wx" });
appendFileSync(env.GITHUB_OUTPUT, `file=${sealed}\n`);
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
try {
main();
} catch (error) {
process.stdout.write(`::error::upload-lane-transcript: ${error instanceof SealError ? error.message : `the transcript could not be sealed (${error.name})`}\n`);
process.exit(1);
if (process.argv[2] === "telemetry") {
try {
sealTelemetry();
} catch (error) {
process.stdout.write(`::warning::upload-lane-transcript: the telemetry is left out: ${error instanceof SealError ? error.message : `it could not be sealed (${error.name})`}\n`);
}
} else {
try {
main();
} catch (error) {
process.stdout.write(`::error::upload-lane-transcript: ${error instanceof SealError ? error.message : `the transcript could not be sealed (${error.name})`}\n`);
process.exit(1);
}
}
}
76 changes: 72 additions & 4 deletions .github/actions/upload-lane-transcript/seal-transcript.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { gunzipSync } from "node:zlib";
import { generate } from "../open-lane-output/keygen.mjs";
import { openRecord, parsePrivateKey } from "../open-lane-output/open.mjs";
import { tempDir } from "../../../lib/test-tmp.mjs";
import { MAX_ARTIFACT_BYTES, sealTelemetry } from "./seal-transcript.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const ACTION = readFileSync(path.join(HERE, "action.yml"), "utf8");
Expand All @@ -28,10 +29,10 @@ function workDir(t) {
return dir;
}

function run(dir, inputs) {
function run(dir, inputs, ...args) {
const output = path.join(dir, "github-output");
writeFileSync(output, "");
const result = spawnSync(process.execPath, [path.join(HERE, "seal-transcript.mjs")], {
const result = spawnSync(process.execPath, [path.join(HERE, "seal-transcript.mjs"), ...args], {
env: { PATH: process.env.PATH, GITHUB_OUTPUT: output, RUNNER_TEMP: dir, GITHUB_RUN_ID: "42", ...stepEnv(inputs) },
encoding: "utf8",
});
Expand Down Expand Up @@ -118,11 +119,78 @@ test("a missing, empty or linked transcript seals nothing; a bad job id fails",
}
});

test("the upload keeps only the sealed file, for 7 days, named by job and attempt", () => {
const TELEMETRY = `${JSON.stringify({ resourceSpans: [{ scopeSpans: [{ spans: [{ name: MARKER }] }] }] })}\n`;

function sealBoth(t) {
const dir = workDir(t);
const keys = generate();
const inputs = {
"public-key": keys.publicLine,
"execution-file": path.join(dir, "execution.json"),
"otel-file": path.join(dir, "otel.jsonl"),
"job-id": "refine-issue",
};
writeFileSync(inputs["execution-file"], TRANSCRIPT);
writeFileSync(inputs["otel-file"], TELEMETRY);
const transcript = run(dir, inputs);
assert.equal(transcript.status, 0, transcript.stdout + transcript.stderr);
return { dir, keys, inputs, transcript };
}

test("the telemetry seals beside the transcript under its own payload name and opens to the same bytes", (t) => {
const { dir, keys, inputs, transcript } = sealBoth(t);
const result = run(dir, inputs, "telemetry");
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.equal(path.dirname(result.outputs.file), path.dirname(transcript.outputs.file));
assert.equal(path.basename(result.outputs.file), "otel.jsonl.gz.sealed");
const sealed = readFileSync(result.outputs.file);
assert.equal(sealed.includes(Buffer.from(MARKER)), false, "no plaintext in the sealed file");
const keyId = keys.publicLine.split(" ")[1];
const open = (name, runId = "42", jobId = "refine-issue") => gunzipSync(openRecord(parsePrivateKey(keys.privateText), [keyId], name, sealed, runId, jobId)).toString("utf8");
assert.equal(open("otel.jsonl.gz"), TELEMETRY);
assert.throws(() => open("transcript.json.gz"), /does not open/, "bound to its own payload name");
assert.throws(() => open("otel.jsonl.gz", "43"), /does not open/, "bound to its run");
assert.equal(openSealed(transcript.outputs.file, keys.privateText, keyId), TRANSCRIPT, "the transcript is untouched");
assert.doesNotMatch(result.stdout + result.stderr, new RegExp(MARKER));
});

test("no telemetry file seals nothing, and a telemetry failure only warns", (t) => {
const { dir, inputs } = sealBoth(t);
for (const file of ["", path.join(dir, "absent.jsonl")]) {
const result = run(dir, { ...inputs, "otel-file": file }, "telemetry");
assert.equal(result.status, 0, file);
assert.deepEqual(result.outputs, {}, file);
}
const result = run(dir, { ...inputs, "public-key": "x25519 0000000000000000 REPLACE_WITH_PUBLIC_KEY" }, "telemetry");
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.match(result.stdout, /^::warning::upload-lane-transcript: the telemetry is left out: /m);
assert.deepEqual(result.outputs, {});
assert.deepEqual(sealedFiles(dir).filter((f) => f.endsWith("otel.jsonl.gz.sealed")), []);
});

test("telemetry that would take the artifact past the puller's limit is left out with a warning", (t) => {
const { dir, inputs } = sealBoth(t);
const output = path.join(dir, "github-output");
writeFileSync(output, "");
const env = { ...stepEnv(inputs), GITHUB_OUTPUT: output, RUNNER_TEMP: dir, GITHUB_RUN_ID: "42" };
const written = [];
const write = process.stdout.write;
process.stdout.write = (line) => written.push(String(line));
try {
sealTelemetry(env, 1);
} finally {
process.stdout.write = write;
}
assert.match(written.join(""), /::warning::upload-lane-transcript: the telemetry is left out: with it the artifact would pass/);
assert.equal(readFileSync(output, "utf8"), "");
assert.equal(MAX_ARTIFACT_BYTES, 63 * 1024 * 1024);
});

test("the upload keeps only the sealed files, for 7 days, named by job and attempt", () => {
const upload = ACTION.slice(ACTION.indexOf("uses: actions/upload-artifact@"));
assert.match(ACTION, /if: steps\.seal\.outputs\.file != ''\n\s+uses: actions\/upload-artifact@[0-9a-f]{40} /);
assert.match(upload, /^ {8}name: lane-transcript-\$\{\{ inputs\.job-id \}\}-\$\{\{ github\.run_attempt \}\}$/m);
assert.match(upload, /^ {8}path: \$\{\{ steps\.seal\.outputs\.file \}\}$/m);
assert.match(upload, /^ {8}path: \|\n {10}\$\{\{ steps\.seal\.outputs\.file \}\}\n {10}\$\{\{ steps\.seal-otel\.outputs\.file \}\}\n/m);
// The issue's retention: 7 days (cant-fail-ok: the issue's stated contract).
assert.match(upload, /^ {8}retention-days: 7$/m);
assert.match(upload, /^ {8}overwrite: false$/m);
Expand Down
Loading
Loading