From 174d93d82279010d3e9d4453992c1313738e5b0a Mon Sep 17 00:00:00 2001 From: Hugo Richard Date: Mon, 16 Nov 2026 18:44:13 +0000 Subject: [PATCH] feat: open the pull request comment on a report card drawn by an evlog cloud app Amp-Thread-ID: https://ampcode.com/threads/T-01a1dfbd-6bcf-7433-9442-0664b5e62f23 --- .github/workflows/test.yml | 31 +++++++++ README.md | 18 +++++ action.yml | 7 ++ src/lib/cards.mjs | 99 +++++++++++++++++++++++++++ src/lib/event.mjs | 1 + src/lib/inputs.mjs | 8 +++ src/lib/report.mjs | 137 +++++++++++++++++++++++++++++++++++-- src/main.mjs | 10 +++ test/cards.test.mjs | 120 ++++++++++++++++++++++++++++++++ test/inputs.test.mjs | 6 ++ 10 files changed, 433 insertions(+), 4 deletions(-) create mode 100644 src/lib/cards.mjs create mode 100644 test/cards.test.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f1bd0d..bb90c8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,6 +145,37 @@ jobs: echo "$OUTPUTS" | jq -e '.results | fromjson | all(has("delta"))' + + cards: + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + id-token: write + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - id: map + continue-on-error: true + uses: ./ + with: + packages: | + test/fixtures/* + # The preview of evloghq/evlog#1075, which writes the `event` key the + # card reads. Back to the default once a release carries it. + version: https://pkg.pr.new/@evlog/cli@1075 + cards-url: https://evlog-cloud.vercel.app + comment: true + comment-key: cards + annotations: false + - name: The app stored the run and the comment links to it + env: + CARDS_URL: ${{ steps.map.outputs.cards-url }} + run: | + echo "$CARDS_URL" + case "$CARDS_URL" in https://evlog-cloud.vercel.app/r/*) ;; *) exit 1 ;; esac + curl -fsS -o /dev/null -w '%{content_type}\n' "$CARDS_URL/report.svg" | grep -q 'image/svg+xml' + telemetry: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 1a766d1..db75c9b 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ The judgment is the CLI's. The action adds no rule of its own, so `npx evlog map | `summary` | `true` | Write the report to the job summary. | | `comment` | `true` | `true` posts one comment and keeps it updated; `on-failure` only posts when the gate failed, and updates an existing comment either way; `false` posts nothing. Needs `pull-requests: write`; skipped with a notice when the token cannot write (forks). | | `comment-key` | `default` | Keeps separate comments when the action runs more than once on a pull request. | +| `cards-url` | | URL of an evlog cloud app that draws the report as image cards in the comment. Needs `id-token: write`. Preview. | | `telemetry` | `true` | Send action health and scan counters. `false` disables telemetry for both the action and its CLI scans. | | `token` | `${{ github.token }}` | Token for the check run and the comment. | @@ -61,6 +62,7 @@ The judgment is the CLI's. The action adds no rule of its own, so `npx evlog map | `instrumented`, `partial`, `dark` | Entry point counts, every package together. | | `passed` | `true` when every package met the gate, whatever `gate` is set to. | | `results` | JSON, one entry per package: `name`, `framework`, `score`, `status`, `reasons`, `summary`, `delta`, `regressions`, `fixed`. | +| `cards-url` | The run page on the `cards-url` app, empty when no cards were drawn. | ## Recipes @@ -126,6 +128,22 @@ The comment is posted with the workflow's token, so it shows as `github-actions` token: ${{ steps.app.outputs.token }} ``` +### Report cards (preview) + +```yaml +permissions: + contents: read + checks: write + pull-requests: write + id-token: write +# ... + - uses: evloghq/action@v1 + with: + cards-url: https://evlog-cloud.vercel.app +``` + +The action posts the run to the app, which stores it with the base branch's score history and answers with an image card. The comment opens on it: the gate, the score and how it moved, the base branch's score commit by commit and where the pull request lands, then the entry points the pull request changed, with the `log.set()` fields and `log.audit()` actions each one loses or gains. Below it, the comment keeps the gate alert with links into the diff, the wide event diff as text, and every entry point against every check, both folded. The wide event needs an `@evlog/cli` that writes the `event` key. The OIDC token proves the repository to the app, so there is no secret to create. If the app is down or refuses the run, the comment goes out without cards and a notice says why. `https://evlog-cloud.vercel.app` is a proof of concept deployment and may move. + ## Frameworks Everything `evlog map` scans: Nuxt, Nitro, Next.js App Router, TanStack Start, Hono, Express, and Fastify. Detection reads `package.json`; nothing is installed. diff --git a/action.yml b/action.yml index 3f6ee6a..5320aa8 100644 --- a/action.yml +++ b/action.yml @@ -46,6 +46,9 @@ inputs: comment-key: description: Distinguishes comments when the action runs more than once on a pull request. default: default + cards-url: + description: 'URL of an evlog cloud app that draws the report as image cards in the comment. Empty draws none. Needs `id-token: write`, which proves the repository to the app.' + default: '' telemetry: description: Send action and CLI telemetry. No repository names, source, paths or tokens. Also disabled by DO_NOT_TRACK=1 or EVLOG_TELEMETRY=0. default: 'true' @@ -78,6 +81,9 @@ outputs: results: description: JSON array with one entry per package (directory, framework, score, delta, summary, regressions, fixed). value: ${{ steps.run.outputs.results }} + cards-url: + description: The run page on the `cards-url` app, empty when no cards were drawn. + value: ${{ steps.run.outputs.cards-url }} runs: using: composite @@ -100,4 +106,5 @@ runs: INPUT_COMMENT_KEY: ${{ inputs.comment-key }} INPUT_TOKEN: ${{ inputs.token }} INPUT_TELEMETRY: ${{ inputs.telemetry }} + INPUT_CARDS_URL: ${{ inputs.cards-url }} GITHUB_ACTION_REF: ${{ github.action_ref }} diff --git a/src/lib/cards.mjs b/src/lib/cards.mjs new file mode 100644 index 0000000..32c6413 --- /dev/null +++ b/src/lib/cards.mjs @@ -0,0 +1,99 @@ +/** + * Report cards: the run goes to the evlog cloud app, which stores it and + * renders it as images the comment embeds. The comment's Markdown stays the + * source of truth; the cards draw the same numbers. + */ + +const AUDIENCE = 'https://evlog.cloud' + +function checkCell(check) { + return check.suppressed ? 'silenced' : check.status +} + +/** The sensitivity categories the CLI found, `money` out of `money: path says "checkout"`. */ +function sensitivity(route) { + if (route.sensitivity.level === 'none') return [] + return [...new Set(route.sensitivity.reasons.map(reason => reason.split(':')[0]))] +} + +/** What the app stores for one run: the CLI's JSON per package, trimmed to what a card draws. */ +export function runPayload(results, context) { + const { repository, serverUrl, sha, refName, pullRequest, cliVersion, minScore, gate = true } = context + return { + repository, + serverUrl, + sha: pullRequest?.headSha ?? sha, + branch: pullRequest?.base ?? refName, + pullRequest: pullRequest && { number: pullRequest.number, baseSha: pullRequest.baseSha, headSha: pullRequest.headSha }, + cliVersion, + minScore, + gate, + packages: results.map(result => ({ + name: result.name === '.' ? result.projectName : result.name, + framework: result.framework, + score: result.score, + status: result.status, + reasons: result.reasons, + summary: result.summary, + baseScore: result.baseline?.baselineScore, + regressions: result.baseline?.regressions ?? [], + fixed: result.baseline?.fixed ?? [], + routes: result.routes.map(route => ({ + id: route.id, + method: route.method, + path: route.path, + file: route.file, + line: route.handler?.line, + score: route.score, + baseScore: result.baselineRoutes?.[route.id], + sensitivity: sensitivity(route), + checks: Object.fromEntries(Object.entries(route.checks).map(([id, check]) => [id, checkCell(check)])), + messages: Object.fromEntries(Object.entries(route.checks).filter(([, check]) => check.status === 'fail' && check.message).map(([id, check]) => [id, check.message])), + event: route.event, + baseEvent: result.baselineEvents?.[route.id], + })), + })), + } +} + +/** + * A GitHub Actions OIDC token for the app, which proves the repository the + * run comes from. Only there with `id-token: write`. + */ +async function oidcToken(env, fetchFn) { + if (!env.ACTIONS_ID_TOKEN_REQUEST_URL || !env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) return undefined + const url = `${env.ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${encodeURIComponent(AUDIENCE)}` + const response = await fetchFn(url, { headers: { authorization: `Bearer ${env.ACTIONS_ID_TOKEN_REQUEST_TOKEN}` } }) + if (!response.ok) return undefined + return (await response.json()).value +} + +/** + * Post the run. Returns the run page and its cards, or why there are none: + * the cards add to the comment, so an app that is down or refuses the run is + * a notice and the comment goes out without them. + */ +export async function publishCards({ endpoint, payload, env = process.env, fetchFn = fetch }) { + const token = await oidcToken(env, fetchFn) + let response + try { + response = await fetchFn(`${endpoint.replace(/\/$/, '')}/api/runs`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...(token ? { authorization: `Bearer ${token}` } : {}) }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(10_000), + }) + } catch (error) { + return { outcome: 'failed', reason: `${endpoint} unreachable: ${error.message}` } + } + if (response.status === 401 && !token) return { outcome: 'skipped', reason: 'the cards need `id-token: write` to prove the repository' } + if (!response.ok) return { outcome: 'failed', reason: `${endpoint} answered ${response.status}` } + return { outcome: 'created', ...(await response.json()) } +} + +const attr = text => String(text).replace(/&/g, '&').replace(/"/g, '"') + +/** One card as GitHub Markdown: the image, linking to the run page. */ +export function picture(card, url) { + return `${attr(card.alt)}` +} diff --git a/src/lib/event.mjs b/src/lib/event.mjs index 7d31c1f..27513f2 100644 --- a/src/lib/event.mjs +++ b/src/lib/event.mjs @@ -14,6 +14,7 @@ export function readEvent(env = process.env) { serverUrl: env.GITHUB_SERVER_URL ?? 'https://github.com', workspace: env.GITHUB_WORKSPACE ?? process.cwd(), sha: env.GITHUB_SHA ?? '', + refName: env.GITHUB_REF_NAME ?? '', runId: env.GITHUB_RUN_ID ?? '', outputFile: env.GITHUB_OUTPUT, summaryFile: env.GITHUB_STEP_SUMMARY, diff --git a/src/lib/inputs.mjs b/src/lib/inputs.mjs index 43b9908..b916c60 100644 --- a/src/lib/inputs.mjs +++ b/src/lib/inputs.mjs @@ -26,6 +26,13 @@ function integer(name, raw, { min, max }) { return value } +function url(name, raw) { + const value = raw.trim() + if (value === '') return undefined + if (!/^https?:\/\/[^\s/]+/.test(value)) throw new InputError(`Input \`${name}\` must be an http(s) URL, got "${raw}"`) + return value +} + function commentMode(raw) { const mode = COMMENT_MODES[raw.trim().toLowerCase() || 'true'] if (!mode) throw new InputError(`Input \`comment\` must be true, false or on-failure, got "${raw}"`) @@ -71,6 +78,7 @@ export function readInputs(env = process.env) { summary: bool('summary', get('SUMMARY') || 'true'), comment: commentMode(get('COMMENT')), commentKey: get('COMMENT_KEY').trim() || 'default', + cardsUrl: url('cards-url', get('CARDS_URL')), token: get('TOKEN'), } } diff --git a/src/lib/report.mjs b/src/lib/report.mjs index 5901ed9..772ffa2 100644 --- a/src/lib/report.mjs +++ b/src/lib/report.mjs @@ -8,6 +8,7 @@ * judgment made here is which entry points to list first. */ import { createHash } from 'node:crypto' +import { picture } from './cards.mjs' const DOCS = 'https://evlog.dev' @@ -132,12 +133,14 @@ const quote = lines => lines.map(line => `> ${line}`) /** * @param results one entry per package, from `main.mjs`; `name` is how the * package is shown, `path` where it sits in the repository, for links; - * `baselineRoutes` maps entry point ids to their score at the base + * `baselineRoutes` maps entry point ids to their score at the base, + * `baselineEvents` to their wide event there * @param context repository, sha, pull request and server for links; * cliVersion for the footer; baselineRef for the verdict; minScore and gate * for what the gate did */ export function renderReport(results, context) { + if (context.cards) return renderCardReport(results, context) const { gate = true, minScore } = context const lines = [headline(results), ''] const line = verdict(results, context) @@ -172,9 +175,7 @@ export function renderReport(results, context) { lines.push('', ...quote(['[!NOTE]', `No ${code('min-score')} set. Add ${code(`min-score: ${lowest(results)}`)} to keep today's score from dropping.`])) } - const priorities = results.flatMap(result => prioritize(result.routes).slice(0, 3).map(route => ({ result, route }))) - .sort((a, b) => a.route.score - b.route.score) - .slice(0, 5) + const priorities = fixFirst(results) if (priorities.length > 0) { lines.push('', ` 0 ? '' : ' open'}>Fix first (${priorities.length})`, '') for (const { result, route } of priorities) { @@ -191,6 +192,134 @@ export function renderReport(results, context) { return lines.join('\n') } +/** Up to five entry points to fix, three per package at most, lowest score first. */ +function fixFirst(results) { + return results.flatMap(result => prioritize(result.routes).slice(0, 3).map(route => ({ result, route }))) + .sort((a, b) => a.route.score - b.route.score) + .slice(0, 5) +} + +/** Why the gate failed, in one line: the checks lost, the score under `min-score`. */ +function gateLine(results, { gate = true, baselineRef }) { + const regressions = results.reduce((sum, result) => sum + (result.baseline?.regressions.length ?? 0), 0) + const parts = [] + if (regressions > 0) parts.push(`${plural(regressions, 'check')} lost against ${code(baselineRef ?? 'the base')}`) + const below = results.flatMap(result => result.reasons).find(reason => reason.startsWith('below')) + if (below) parts.push(below.replace('below --min-score', 'below `min-score`')) + if (parts.length === 0) parts.push(`score dropped against ${code(baselineRef ?? 'the base')}`) + return `${gate ? 'Gate failed' : 'Gate would fail'}: ${parts.join(', ')}${gate ? '' : ' (`gate: false`, the step stays green)'}` +} + +const EMPTY_EVENT = { fields: [], audit: [] } +const MAX_KEPT = 3 + +/** An event's lines as the diff prints them: field paths, then audit actions. */ +function eventLines(event = EMPTY_EVENT) { + return [...event.fields, ...event.audit.map(action => `audit "${action}"`)] +} + +/** + * Entry points whose wide event this pull request changes, from the `event` + * key `evlog map` writes on the base and on the head. An entry point that is + * new in the pull request has nothing to compare against and is left out. + */ +export function eventChanges(results) { + const changes = results.flatMap(result => result.routes.flatMap((route) => { + if (!result.baselineEvents || !(route.id in result.baselineEvents)) return [] + const base = result.baselineEvents[route.id] + const before = eventLines(base) + const after = eventLines(route.event) + const gone = before.filter(line => !after.includes(line)) + const added = after.filter(line => !before.includes(line)) + if (gone.length === 0 && added.length === 0) return [] + return [{ result, route, before, after, gone, added, dark: !route.event && Boolean(base) }] + })) + const sensitive = change => (change.route.sensitivity.level === 'none' ? 1 : 0) + return changes.sort((a, b) => (b.gone.length > 0) - (a.gone.length > 0) || sensitive(a) - sensitive(b) || b.gone.length - a.gone.length) +} + +/** + * The changes as a `diff` block GitHub highlights: one hunk per entry point, + * its score movement in the hunk header, dropped lines in red, new ones in + * green, and a few unchanged lines for context. + */ +export function eventDiff(changes, { multi = false } = {}) { + const hunks = changes.map(({ result, route, before, after, added, dark }) => { + const name = multi && result.path !== '.' ? `${result.path} ` : '' + const base = result.baselineRoutes?.[route.id] + const score = base === undefined || base === route.score ? `${route.score}` : `${base} → ${route.score}` + const lines = [`@@ ${name}${routeLabel(route)} · ${score}${dark ? ' · no wide event after merge' : ''} @@`] + let kept = 0 + for (const line of before) { + if (!after.includes(line)) lines.push(`- ${line}`) + else if (kept++ < MAX_KEPT) lines.push(` ${line}`) + } + if (kept > MAX_KEPT) lines.push(` … ${kept - MAX_KEPT} more unchanged`) + for (const line of added) lines.push(`+ ${line}`) + return lines.join('\n') + }) + return ['```diff', hunks.join('\n\n'), '```'].join('\n') +} + +/** What merging does to production logs, in one sentence, or nothing when it drops nothing. */ +function mergeLine(changes) { + const gone = changes.flatMap(change => change.gone) + const audits = gone.filter(line => line.startsWith('audit ')).length + const lost = [gone.length > audits ? plural(gone.length - audits, 'field') : '', audits > 0 ? plural(audits, 'audit action') : ''].filter(Boolean) + const dark = changes.filter(change => change.dark).map(change => code(routeLabel(change.route))) + const parts = [] + if (lost.length > 0) parts.push(`drops ${lost.join(' and ')} from production logs`) + if (dark.length > 0) parts.push(`takes ${dark.join(', ')} dark`) + return parts.length > 0 ? `Merging this ${parts.join(' and ')}.` : undefined +} + +/** + * The comment when the evlog cloud app drew the report card. The card opens + * the comment with the score, its trend and the entry points that changed. + * The Markdown keeps what an image cannot do: the gate as an alert with links + * into the diff, the wide event diff as text, and the full grid folded away. + */ +function renderCardReport(results, context) { + const { minScore, cards } = context + const card = name => cards.cards.find(item => item.name === name) + const changes = eventChanges(results) + const merge = mergeLine(changes) + const lines = [] + if (card('report')) lines.push(picture(card('report'), cards.url), '') + + const seen = new Set() + const links = results.flatMap(result => (result.baseline?.regressions ?? []).map((regression) => { + const route = result.routes.find(item => item.id === regression.routeId) + const key = `${result.path}:${regression.routeId}` + if (!route || seen.has(key)) return undefined + seen.add(key) + return fileLink(result, route, context, { diff: true }) + })).filter(Boolean) + if (results.some(result => result.status === 'failed')) { + lines.push(...quote(['[!CAUTION]', ...(merge ? [`**${merge}**`, gateLine(results, context)] : [`**${gateLine(results, context)}**`]), ...(links.length > 0 ? [links.join(' · ')] : [])]), '') + } else if (merge) { + lines.push(...quote(['[!WARNING]', `**${merge}**`]), '') + } + + if (changes.length > 0) { + lines.push(`
Wide event diff, ${plural(changes.length, 'entry point')}`, '', eventDiff(changes, { multi: results.length > 1 }), '', '
') + } + const total = results.reduce((sum, result) => sum + result.routes.length, 0) + const priorities = fixFirst(results) + lines.push(`
All ${plural(total, 'entry point')}${priorities.length > 0 ? ', fix first' : ''}`, '') + if (card('matrix')) lines.push(picture(card('matrix'), cards.url), '') + for (const { result, route } of priorities) { + lines.push(`- ${code(routeLabel(route))} ${fileLink(result, route, context)}: ${failedChecks(route).map(code).join(', ')}`) + } + lines.push('', '
', '') + + if (minScore === undefined) { + lines.push(...quote(['[!NOTE]', `No ${code('min-score')} set. Add ${code(`min-score: ${lowest(results)}`)} to keep today's score from dropping.`]), '') + } + lines.push(footer(context)) + return lines.join('\n') +} + function footer(context) { const version = context.cliVersion ? ` v${context.cliVersion}` : '' return `[evlog map](${DOCS}/cli/map)${version} · [how the score works](${DOCS}/cli/scoring) · [what each check expects](${DOCS}/cli/rules)` diff --git a/src/main.mjs b/src/main.mjs index 2ae6b5c..d1df20d 100644 --- a/src/main.mjs +++ b/src/main.mjs @@ -1,6 +1,7 @@ import { appendFileSync } from 'node:fs' import { relative, resolve } from 'node:path' import { baselineFor, checkoutBase } from './lib/baseline.mjs' +import { publishCards, runPayload } from './lib/cards.mjs' import { createCheckRun } from './lib/checks.mjs' import { gateArgs, packageSpec, runMap } from './lib/cli.mjs' import { upsertComment } from './lib/comment.mjs' @@ -102,6 +103,7 @@ async function main(inputs, telemetry) { summary: json.summary, baseline: json.baseline, baselineRoutes: scanned ? Object.fromEntries(scanned.map.routes.map(route => [route.id, route.score])) : undefined, + baselineEvents: scanned ? Object.fromEntries(scanned.map.routes.map(route => [route.id, route.event])) : undefined, status: scan.status === 0 ? 'passed' : 'failed', reasons: failed, }) @@ -117,6 +119,14 @@ async function main(inputs, telemetry) { telemetry?.set(scanFields(outputs)) telemetry?.set({ checkOutcome: 'disabled', commentOutcome: 'disabled' }) const context = { ...event, cliVersion, baselineRef, minScore: inputs.minScore, gate: inputs.gate } + if (inputs.cardsUrl) { + const cards = await publishCards({ endpoint: inputs.cardsUrl, payload: runPayload(results, context) }) + if (cards.outcome === 'created') { + context.cards = cards + log(`evlog: cards ${cards.url}`) + } else notice(`report cards skipped: ${cards.reason}`) + } + outputs['cards-url'] = context.cards?.url ?? '' const report = renderReport(results, context) if (inputs.summary && event.summaryFile) appendFileSync(event.summaryFile, `${renderSummary(results, context)}\n`) diff --git a/test/cards.test.mjs b/test/cards.test.mjs new file mode 100644 index 0000000..23711a7 --- /dev/null +++ b/test/cards.test.mjs @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { picture, publishCards, runPayload } from '../src/lib/cards.mjs' +import { renderReport } from '../src/lib/report.mjs' + +const result = { + name: '.', path: '.', framework: 'nuxt', projectName: 'shop', score: 56, status: 'failed', reasons: ['regressed'], + summary: { instrumented: 1, partial: 0, dark: 1, exempt: 0 }, + routes: [{ + id: 'r1', method: 'POST', path: '/checkout', file: 'server/api/checkout.post.ts', handler: { line: 3 }, score: 75, + sensitivity: { level: 'high', reasons: ['money: path says "checkout"', 'money: calls stripe'] }, + checks: { 'wide-event': { status: 'pass' }, 'audit': { status: 'fail', message: 'sensitive route without log.audit()' }, 'context': { status: 'fail', suppressed: true } }, + event: { fields: ['cart.items'], audit: [] }, + }], + baseline: { baselineScore: 68, delta: -12, regressions: [{ routeId: 'r1', path: '/checkout', method: 'POST', file: 'server/api/checkout.post.ts', check: 'audit', to: 'fail' }], fixed: [] }, + baselineRoutes: { r1: 100 }, + baselineEvents: { r1: { fields: ['user.id', 'cart.items'], audit: ['checkout.completed'] } }, +} +const pullRequest = { number: 7, base: 'main', baseSha: 'b1', headSha: 'h1' } +const context = { repository: 'acme/shop', serverUrl: 'https://github.com', sha: 'm1', refName: '7/merge', pullRequest, cliVersion: '0.11.2', minScore: 60 } + +const cards = { + outcome: 'created', + url: 'https://evlog.cloud/r/run1', + cards: ['report', 'matrix'].map(name => ({ name, alt: `${name} card`, url: `https://evlog.cloud/r/run1/${name}.svg` })), +} + +describe('runPayload', () => { + it('is the pull request against its base branch, with each entry point as the cards draw it', () => { + const payload = runPayload([result], context) + assert.equal(payload.branch, 'main') + assert.equal(payload.sha, 'h1') + assert.deepEqual(payload.pullRequest, { number: 7, baseSha: 'b1', headSha: 'h1' }) + assert.equal(payload.packages[0].name, 'shop') + assert.equal(payload.packages[0].baseScore, 68) + assert.deepEqual(payload.packages[0].routes[0], { + id: 'r1', method: 'POST', path: '/checkout', file: 'server/api/checkout.post.ts', line: 3, score: 75, baseScore: 100, + sensitivity: ['money'], + checks: { 'wide-event': 'pass', 'audit': 'fail', 'context': 'silenced' }, + messages: { audit: 'sensitive route without log.audit()' }, + event: { fields: ['cart.items'], audit: [] }, + baseEvent: { fields: ['user.id', 'cart.items'], audit: ['checkout.completed'] }, + }) + }) + + it('is the pushed branch on a push', () => { + const payload = runPayload([{ ...result, baseline: undefined, baselineRoutes: undefined }], { ...context, pullRequest: undefined, refName: 'main' }) + assert.equal(payload.branch, 'main') + assert.equal(payload.sha, 'm1') + assert.equal(payload.pullRequest, undefined) + assert.equal(payload.packages[0].baseScore, undefined) + assert.deepEqual(payload.packages[0].regressions, []) + }) +}) + +describe('publishCards', () => { + const oidcEnv = { ACTIONS_ID_TOKEN_REQUEST_URL: 'https://token.actions/req?x=1', ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'req-token' } + + it('proves the repository with an OIDC token for the app audience', async () => { + const calls = [] + const fetchFn = async (url, init = {}) => { + calls.push({ url, init }) + if (new URL(url).host === 'token.actions') return { ok: true, status: 200, json: async () => ({ value: 'oidc-jwt' }) } + return { ok: true, status: 200, json: async () => ({ id: 'run1', url: cards.url, cards: cards.cards }) } + } + const outcome = await publishCards({ endpoint: 'https://evlog.cloud/', payload: { repository: 'acme/shop' }, env: oidcEnv, fetchFn }) + assert.equal(calls[0].url, 'https://token.actions/req?x=1&audience=https%3A%2F%2Fevlog.cloud') + assert.equal(calls[1].url, 'https://evlog.cloud/api/runs') + assert.equal(calls[1].init.headers.authorization, 'Bearer oidc-jwt') + assert.equal(outcome.outcome, 'created') + assert.equal(outcome.cards.length, 2) + }) + + it('names the missing permission when the app refuses a run without a token', async () => { + const fetchFn = async () => ({ ok: false, status: 401 }) + const outcome = await publishCards({ endpoint: 'https://evlog.cloud', payload: {}, env: {}, fetchFn }) + assert.deepEqual(outcome, { outcome: 'skipped', reason: 'the cards need `id-token: write` to prove the repository' }) + }) + + it('reports an unreachable app instead of throwing', async () => { + const fetchFn = async () => { throw new Error('ECONNREFUSED') } + const outcome = await publishCards({ endpoint: 'https://evlog.cloud', payload: {}, env: {}, fetchFn }) + assert.equal(outcome.outcome, 'failed') + assert.match(outcome.reason, /unreachable: ECONNREFUSED/) + }) +}) + +describe('the report with cards', () => { + it('opens on the report card, then says what merging drops and why the gate failed, with the diff folded', () => { + const md = renderReport([result], { ...context, baselineRef: 'main', cards }) + assert.match(md, /^report card<\/a>\n\n> \[!CAUTION\]\n> \*\*Merging this drops 1 field and 1 audit action from production logs\.\*\*\n> Gate failed: 1 check lost against `main`\n> \[`server\/api\/checkout\.post\.ts:3`\]\(https:\/\/github\.com\/acme\/shop\/pull\/7\/files#diff-[0-9a-f]+R3\)\n\n
Wide event diff, 1 entry point<\/summary>\n/) + assert.ok(md.includes('```diff\n@@ POST /checkout · 100 → 75 @@\n- user.id\n cart.items\n- audit "checkout.completed"\n```')) + assert.match(md, /
All 1 entry point, fix first<\/summary>\n\n { + const routes = [{ ...result.routes[0], event: undefined }, { ...result.routes[0], id: 'r2', path: '/new' }] + const md = renderReport([{ ...result, routes }], { ...context, baselineRef: 'main', cards }) + assert.match(md, /Merging this drops 2 fields and 1 audit action from production logs and takes `POST \/checkout` dark\./) + assert.match(md, /@@ POST \/checkout · 100 → 75 · no wide event after merge @@/) + assert.doesNotMatch(md, /@@ POST \/new/) + }) + + it('says the gate would fail when it is off, and draws no alert when it passed', () => { + assert.match(renderReport([result], { ...context, gate: false, baselineRef: 'main', cards }), /> Gate would fail: 1 check lost against `main` \(`gate: false`, the step stays green\)\n/) + assert.doesNotMatch(renderReport([{ ...result, status: 'passed', reasons: [] }], { ...context, cards }), /\[!CAUTION\]/) + }) + + it('is unchanged without cards', () => { + const md = renderReport([result], { ...context, baselineRef: 'main' }) + assert.doesNotMatch(md, / { + assert.match(picture({ alt: 'a "b" & c', url: 'l' }, 'u'), /alt="a "b" & c"/) + }) +}) diff --git a/test/inputs.test.mjs b/test/inputs.test.mjs index 40697ab..329e191 100644 --- a/test/inputs.test.mjs +++ b/test/inputs.test.mjs @@ -21,6 +21,7 @@ describe('readInputs', () => { summary: true, comment: 'always', commentKey: 'default', + cardsUrl: undefined, token: 't', }) }) @@ -54,6 +55,11 @@ describe('readInputs', () => { assert.throws(() => readInputs(env({ INPUT_TELEMETRY: 'yes' })), /`telemetry` must be true or false/) }) + it('reads the cards URL and rejects anything else', () => { + assert.equal(readInputs(env({ INPUT_CARDS_URL: ' https://evlog.cloud ' })).cardsUrl, 'https://evlog.cloud') + assert.throws(() => readInputs(env({ INPUT_CARDS_URL: 'evlog.cloud' })), /`cards-url` must be an http\(s\) URL/) + }) + it('rejects a boolean that is neither true nor false', () => { assert.throws(() => readInputs(env({ INPUT_GATE: 'yes' })), /`gate` must be true or false/) })