Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions bot/bot.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
branchFor,
daysRemaining,
hasMetadataMarker,
logWarning,
markdownCode,
markdownLink,
markdownText,
Expand All @@ -37,6 +38,7 @@ import {
import { assertValidPlanDocument } from '../lib/validate-document.mjs'
import { captureWindowFor, lifecycleFor } from '../lib/feeds.mjs'
import { parseCliArgs } from '../lib/cli.mjs'
import { printable, printableLines } from '../lib/printable.mjs'
import { EVAL_RESULTS_SCHEMA as EVAL_SCHEMA, assertValidEvalResults } from '../lib/receipts.mjs'
import { loadConfig } from './lib/config.mjs'
import { downloadFeeds } from './lib/feeds.mjs'
Expand Down Expand Up @@ -164,7 +166,7 @@ const validateRepo = repo => {
return repo
}

const spawnPlan = ({ workDir, config, configPath = null, feedsDir, checkerPath = CHECKER, warn = console.error }) => {
const spawnPlan = ({ workDir, config, configPath = null, feedsDir, checkerPath = CHECKER, warn = logWarning }) => {
const args = [checkerPath, 'plan', '.']
if (configPath) args.push('--config', configPath)
else {
Expand Down Expand Up @@ -508,7 +510,7 @@ export const buildIssueBody = ({ group, now = new Date() }) => {
...(group.channel ? { channel: group.channel } : {}),
}
const evidence = group.issues
.map(issue => `- ${markdownText(`${repoPath(issue.file, group.root || '.')}:${issue.line} - ${issue.reason}${issue.matched ? ` (${issue.matched})` : ''}`)}`)
.map(issue => `- ${markdownCode(`${repoPath(issue.file, group.root || '.')}:${issue.line}`)} - ${markdownText(issue.reason)}${issue.matched ? ` (${markdownCode(issue.matched)})` : ''}`)
.join('\n')
const sections = [
metadataLine(metadata),
Expand Down Expand Up @@ -740,7 +742,7 @@ export const evaluatePlan = async ({
fetchImpl = globalThis.fetch,
checkerPath = CHECKER,
vendoredFeeds = VENDORED_FEEDS,
warn = message => console.error(message),
warn = logWarning,
now = new Date(),
} = {}) => {
if (!planFile) throw new Error('--plan-file is required for evaluate')
Expand Down Expand Up @@ -1285,7 +1287,7 @@ export const formatDecisions = (decisions, { degraded = false } = {}) => {
if (record.branch) lines.push(` branch: ${record.branch}`)
if (record.body) lines.push(' body:', record.body.split('\n').map(line => ` ${line}`).join('\n'))
}
return lines.join('\n')
return printableLines(lines.join('\n'))
}

const BLOCKED_ACTIONS = new Set(['conflict', 'eval-failed', 'label-unavailable', 'report-only', 'stand-down'])
Expand Down Expand Up @@ -1348,7 +1350,7 @@ export const runBot = async ({
evalResultsFile = null,
evalCommandOverride = null,
now = new Date(),
warn = message => console.error(message),
warn = logWarning,
} = {}) => {
if (deprecatedInlineEval || evalEnabled) {
throw new Error('inline --eval was removed because publication is write-capable; run `model-eol-bot evaluate` in a read-only job and pass its manifest with MODEL_EOL_EVAL_RESULTS_FILE')
Expand Down Expand Up @@ -1488,6 +1490,8 @@ export const runBot = async ({
}
}

export const evalResultLines = results => results.map(item => printable(`${item.status} ${item.publisher}/${item.id}`))

export const main = async (argv = process.argv.slice(2), env = process.env) => {
if (argv[0] === 'evaluate') {
const options = parseEvaluateArgs(argv.slice(1))
Expand All @@ -1504,7 +1508,7 @@ export const main = async (argv = process.argv.slice(2), env = process.env) => {
commandOverride: env.MODEL_EOL_EVAL_COMMAND || null,
})
appendEvalSummary(env.GITHUB_STEP_SUMMARY, result.artifact)
for (const item of result.artifact.results) console.log(`${item.status} ${item.publisher}/${item.id}`)
for (const line of evalResultLines(result.artifact.results)) console.log(line)
return 0
}
const options = parseArgs(argv)
Expand All @@ -1531,7 +1535,7 @@ export const main = async (argv = process.argv.slice(2), env = process.env) => {
if (options.dryRun) console.log(formatDecisions(result.decisions, { degraded: result.degraded }))
else {
if (result.degraded) console.log('model-eol status: degraded - vendored feed fallback enabled; no GitHub writes')
for (const item of result.decisions) console.log(`${item.action} ${item.group.publisher}/${item.group.kind === 'model' ? item.group.id : item.group.subject}`)
for (const item of result.decisions) console.log(printable(`${item.action} ${item.group.publisher}/${item.group.kind === 'model' ? item.group.id : item.group.subject}`))
}
appendDecisionSummary(env.GITHUB_STEP_SUMMARY, result.decisions, { degraded: result.degraded })
return result.degraded || result.decisions.some(item => BLOCKED_ACTIONS.has(item.action)) ? 1 : 0
Expand All @@ -1548,7 +1552,7 @@ const invokedFile = (() => {

if (invokedFile === fs.realpathSync(fileURLToPath(import.meta.url))) {
main().then(code => process.exit(code)).catch(error => {
console.error(`model-eol bot: ${error.message}`)
console.error(printableLines(`model-eol bot: ${error.message}`))
process.exit(2)
})
}
35 changes: 23 additions & 12 deletions bot/lib/common.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { sha256 } from '../../lib/digests.mjs'
import { printable, printableLines } from '../../lib/printable.mjs'

export { matchesAnyGlob, matchesGlob } from '../../lib/glob.mjs'
export { activeWaiver, itemDigest, repoPath, sha256, sortedJsonDigest, stableJson } from '../../lib/digests.mjs'
Expand All @@ -25,7 +26,8 @@ export const daysRemaining = (shutdown, now = new Date()) => {
return Math.ceil((new Date(`${shutdown}T00:00:00Z`) - now) / 86400000)
}

export const metadataLine = metadata => `<!-- model-eol ${JSON.stringify(metadata).replaceAll('-->', '\\u002d\\u002d\\u003e')} -->`
// HTML also closes a comment at --!>, and parseMetadata's . stops at U+2028 and U+2029
export const metadataLine = metadata => `<!-- model-eol ${JSON.stringify(metadata).replaceAll('-->', '\\u002d\\u002d\\u003e').replaceAll('<', '\\u003c').replaceAll('>', '\\u003e').replace(/[\u2028\u2029]/g, character => `\\u${character.charCodeAt(0).toString(16)}`)} -->`

export const hasMetadataMarker = body => String(body ?? '').includes('<!-- model-eol')

Expand Down Expand Up @@ -57,7 +59,7 @@ export const parseMetadata = body => {
}

export const staleWorkComment = (kind, waiver = null) => waiver
? `model-eol is closing this bot-owned ${kind} because an active waiver makes its finding non-actionable. The waiver expires ${waiver.expires}, is owned by ${markdownText(waiver.owner)}, and records: ${markdownText(waiver.reason)}`
? `model-eol is closing this bot-owned ${kind} because an active waiver makes its finding non-actionable. The waiver expires ${markdownText(waiver.expires)}, is owned by ${markdownCode(waiver.owner)}, and records: ${markdownCode(waiver.reason)}`
: `model-eol is closing this bot-owned ${kind} because its finding is no longer actionable on the repository's current default branch. The reference may have been removed, ignored, retracted by the feed, moved to another clock, or disabled by repository configuration.`

const entityMap = new Map([
Expand All @@ -70,24 +72,31 @@ const entityMap = new Map([
[']', '&#93;'],
['(', '&#40;'],
[')', '&#41;'],
['#', '&#35;'],
['#', '#&#8203;'],
['*', '&#42;'],
['_', '&#95;'],
['~', '&#126;'],
['!', '&#33;'],
['@', '@&#8203;'],
['|', '&#124;'],
])
const leadingMarker = new Map([['-', '&#45;'], ['+', '&#43;'], ['=', '&#61;']])

export const markdownText = value => String(value ?? '')
.replaceAll('\r\n', '\n')
.replaceAll('\r', '\n')
.replace(/[&<>`\\[\]()#*_~!]/g, character => entityMap.get(character))
.split('\n')
.map(line => line.replace(/^(\s*)([-+>]\s|\d+\.\s)/, '$1&#45; '))
.join('\n')
const markdownInline = value => printable(String(value ?? '').replace(/\r\n|[\r\n]/g, ' '))
.replace(/[&<>`\\[\]()#*_~!@|]/g, character => entityMap.get(character))
.replace(/^\s+/, '')
.replace(/^[-+=]/, character => leadingMarker.get(character))
.replace(/^(\d+)\./, '$1&#46;')

// one line of text: no block structure, table cells, bare autolinks, or issue references
export const markdownText = value => markdownInline(value)
.replace(/(www)\./gi, '$1&#8203;.')
.replaceAll('/', '/&#8203;')
.replace(/\b(gh)(?=-\d)/gi, '$1&#8203;')

// code spans show entities literally, so drop backticks and line breaks instead of escaping
export const markdownCode = value => {
const text = String(value ?? '').replace(/[\r\n]+/g, ' ').replaceAll('`', '')
const text = printable(String(value ?? '').replace(/[\r\n]+/g, ' ').replaceAll('`', ''))
return `\`${text === '' ? ' ' : text}\``
}

Expand All @@ -97,10 +106,12 @@ export const markdownLink = (label, value) => {
const url = new URL(text)
if (url.protocol !== 'http:' && url.protocol !== 'https:') return markdownText(text)
const href = url.toString().replaceAll('\\', '%5C').replaceAll('(', '%28').replaceAll(')', '%29')
return `[${markdownText(label)}](${href})`
return `[${markdownInline(label === text ? href : label)}](${href})`
} catch {
return markdownText(text)
}
}

export const markdownFenceText = value => String(value ?? '').replaceAll('`', '')

export const logWarning = message => console.error(printableLines(message))
3 changes: 2 additions & 1 deletion bot/lib/feeds.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import os from 'node:os'
import path from 'node:path'

import { assertValidFeed } from '../../lib/validate-feed.mjs'
import { logWarning } from './common.mjs'

const responseText = async response => {
if (typeof response?.text === 'function') return response.text()
Expand All @@ -16,7 +17,7 @@ export const downloadFeeds = async ({
fetchImpl = globalThis.fetch,
vendoredDir,
allowVendoredFallback = false,
warn = console.error,
warn = logWarning,
}) => {
if (!urls.length) return { dir: vendoredDir, cleanup: () => {} }
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-feeds-'))
Expand Down
4 changes: 3 additions & 1 deletion bot/lib/github.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { logWarning } from './common.mjs'

const linkFor = (headers, relation) => {
const value = headers?.get?.('link') || headers?.get?.('Link') || headers?.link || headers?.Link || ''
const match = value.split(',').map(part => part.trim()).find(part => part.endsWith(`rel="${relation}"`))
Expand Down Expand Up @@ -32,7 +34,7 @@ const MODEL_EOL_LABEL = {
}

export class GitHubClient {
constructor({ repo, apiUrl = 'https://api.github.com', token, transport = globalThis.fetch, warn = console.error }) {
constructor({ repo, apiUrl = 'https://api.github.com', token, transport = globalThis.fetch, warn = logWarning }) {
this.repo = repo
this.apiUrl = apiUrl.replace(/\/$/, '')
this.token = token
Expand Down
77 changes: 76 additions & 1 deletion bot/test/run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ import {
runBot,
runPlan,
} from '../bot.mjs'
import { branchFor, itemDigest, markdownCode, metadataLine, parseMetadata, slugFor } from '../lib/common.mjs'
import { branchFor, itemDigest, markdownCode, markdownText, metadataLine, parseMetadata, slugFor, staleWorkComment } from '../lib/common.mjs'
import { loadConfig } from '../lib/config.mjs'
import { downloadFeeds } from '../lib/feeds.mjs'
import { GitHubClient } from '../lib/github.mjs'
import * as botModule from '../bot.mjs'
import { capReport, readReportCapped, reportForBody, runEvalHook } from '../lib/eval.mjs'
import { cloneRepository, deleteRemoteBranch, gitAuthentication, originFor } from '../lib/git.mjs'
import { assertValidEvalResults } from '../../lib/receipts.mjs'
Expand Down Expand Up @@ -433,6 +435,79 @@ const tentativeBotResult = await runBot({
assert(tentativeBotResult.plan.items.length === 0 && tentativeBotResult.decisions.length === 0, 'tentative-floor models produce no migration item or bot work')

assert(markdownCode('custom_hub') === '`custom_hub`' && markdownCode('a`b``c') === '`abc`' && markdownCode('x\r\ny') === '`x y`' && markdownCode('') === '` `' && markdownCode(null) === '` `', 'code spans keep underscores literal and never contain backticks or line breaks')
const hostileMetadata = { schema: 'model-eol.bot/0.1', id: 'o3-deep-research-2025-06-26', publisher: 'openai', shutdown: '2026-07-23', head_sha: null, feed_digest: 'd', replacement_note: 'a --!><img src=https://example.com/p.png> b --> c' }
const hostileLine = metadataLine(hostileMetadata)
assert(!/[<>]/.test(hostileLine.slice('<!--'.length, -'-->'.length)) && parseMetadata(hostileLine)?.replacement_note === hostileMetadata.replacement_note, 'metadata comments carry no raw angle bracket that could close them early and still round-trip')
const lineSeparator = String.fromCharCode(0x2028)
const separatorMetadata = { ...hostileMetadata, replacement_note: `a${lineSeparator}b${String.fromCharCode(0x2029)}c` }
const separatorLine = metadataLine(separatorMetadata)
assert(!separatorLine.includes(lineSeparator) && parseMetadata(separatorLine)?.replacement_note === separatorMetadata.replacement_note, 'metadata comments encode U+2028 and U+2029 so the metadata line still parses')
const hostileText = markdownText('azure/x\n---\n@thossullivan see https://evil.example www.evil.example |a|b| \x1b[2K')
assert(!/[\n\r|\x1b]|@(?!&#8203;)|:\/\/|www\.(?!&#8203;)/i.test(hostileText) && hostileText.includes('@&#8203;thossullivan') && hostileText.includes('&#124;'), 'markdown text stays on one line and neutralizes mentions, table pipes, autolinks, and controls')
const referenceText = markdownText('ping @thossullivan about #1, owner/repo#1, and user@example.com')
assert(referenceText === 'ping @&#8203;thossullivan about #&#8203;1, owner/&#8203;repo#&#8203;1, and user@&#8203;example.com', 'markdown text breaks mentions, issue references, and emails with a zero-width space, which GitHub does not resolve through')
const issuePathText = markdownText('see octocat/Hello-World/issues/1, https://github.com/o/r/pull/2, /issues/3, GH-4, and www.evil.example')
assert(!/\/(?!&#8203;)|www(?!&#8203;)|gh-/i.test(issuePathText), 'markdown text breaks issue paths, GH- references, and www. links before GitHub can link them')
for (const [block, rendered] of [['---', '&#45;--'], [' ---', '&#45;--'], ['+', '&#43;'], ['1. one', '1&#46; one'], [' code', 'code'], ['===', '&#61;==']]) {
assert(markdownText(block) === rendered, `markdown text cannot start a block element: ${JSON.stringify(block)}`)
}
const spoofedSource = buildIssueBody({
group: {
kind: 'issue',
id: 'o3-deep-research-2025-06-26',
subject: 'o3-deep-research-2025-06-26',
publisher: 'openai',
shutdown: '2026-07-23',
via: null,
root: '.',
feedDigest: 'd',
context: { announced: null, notes: [], entry: null },
issues: [{ file: 'a.py', line: 1, reason: 'not-direct-api', status: 'retired', matched: 'o3-deep-research', sources: ['https://evil.example\\@github.com/advisory'] }],
},
now: new Date('2026-08-01T00:00:00Z'),
})
assert(spoofedSource.includes('- [https://evil.example/@&#8203;github.com/advisory](https://evil.example/@github.com/advisory)'),'source link labels show the normalized URL the link actually opens')
const hostileIssue = buildIssueBody({
group: {
kind: 'issue',
id: 'o3-deep-research-2025-06-26',
subject: 'o3-deep-research-2025-06-26',
publisher: 'openai',
shutdown: '2026-07-23',
via: null,
root: '.',
feedDigest: 'd',
context: { announced: null, notes: [], entry: null },
issues: [{ file: 'azure/x\n---\n@thossullivan see https://evil.example now.py', line: 3, reason: 'not-direct-api', status: 'retired', matched: 'o3-deep-research' }],
},
now: new Date('2026-08-01T00:00:00Z'),
})
const hostileIssueLines = hostileIssue.split('\n')
const evidenceLines = hostileIssueLines.slice(hostileIssueLines.indexOf('## Evidence') + 1, hostileIssueLines.indexOf('## Sources') - 1)
assert(evidenceLines.length === 1 && evidenceLines[0] === '- `azure/x --- @thossullivan see https:/evil.example now.py:3` - not-direct-api (`o3-deep-research`)' && !hostileIssueLines.includes('---'), 'issue evidence code-spans repository paths so they cannot add rules, mentions, or links')
const waivedComment = staleWorkComment('issue', { expires: '2099-01-01', owner: '@thossullivan', reason: 'see [x](https://evil.example)\n# h' })
assert(waivedComment.includes('owned by `@thossullivan`') && waivedComment.includes('records: `see [x](https://evil.example) # h`') && !waivedComment.includes('\n'), 'stale-work comments code-span waiver owner and reason')
const decisionLog = formatDecisions([{ action: 'create', group: { kind: 'issue', publisher: 'openai', subject: 'x\n::error::forged' }, body: 'line\n::warning title=forged::x\n ##[error]legacy\n\x1b[2K' }])
assert(decisionLog.startsWith('model-eol decision table\n') && !decisionLog.split('\n').some(line => line.trimStart().startsWith('::') || line.includes('##[') || line.includes('\x1b')), 'dry-run decision tables cannot open workflow commands')
const opensWorkflowCommand = text => String(text).split('\n').some(line => line.trimStart().startsWith('::') || line.includes('##['))
const evalLines = botModule.evalResultLines?.([{ status: 'pass', publisher: 'p\n::warning title=forged::eval', id: 'x ##[error]y' }]) ?? []
assert(evalLines.length === 1 && !opensWorkflowCommand(evalLines.join('\n')), 'evaluate result lines escape plan publisher and id')
const capturedWarnings = []
const originalConsoleError = console.error
console.error = (...values) => capturedWarnings.push(values.join(' '))
try {
await downloadFeeds({
urls: ['https://feeds.invalid/forged.json'],
vendoredDir: path.join(root, 'feeds'),
allowVendoredFallback: true,
fetchImpl: async () => ({ status: 200, ok: true, text: async () => JSON.stringify({ spec: 'x\n::warning title=forged::feed', publisher: 'p', generated: '2026-08-01T00:00:00Z', models: [] }) }),
})
const labelClient = new GitHubClient({ repo: 'example/repo', token: 't', transport: async () => ({ status: 500, ok: false, data: 'x\n::warning title=forged::api ##[error]y' }) })
await labelClient.ensureModelEolLabel()
} finally {
console.error = originalConsoleError
}
assert(capturedWarnings.length === 2 && capturedWarnings.every(message => message.includes('forged') && !opensWorkflowCommand(message)), 'default feed and GitHub API warnings cannot open workflow commands')
const clockDigestFeeds = path.join(tempRoot, 'clock-digest-feeds')
fs.mkdirSync(clockDigestFeeds)
const clockDigestRows = status => [
Expand Down
Loading
Loading