diff --git a/.github/workflows/production-smoke.yml b/.github/workflows/production-smoke.yml new file mode 100644 index 0000000..1cd9e7b --- /dev/null +++ b/.github/workflows/production-smoke.yml @@ -0,0 +1,77 @@ +name: Production continuity verification + +on: + workflow_dispatch: + inputs: + metered_mode: + description: Optional single paid canary for this run + required: true + default: none + type: choice + options: + - none + - agent + - diagnose + schedule: + - cron: "17 4 * * *" + +permissions: + contents: read + +concurrency: + group: production-continuity + cancel-in-progress: false + +jobs: + verify-production: + name: clawfix.dev contract + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + CLAWFIX_API_TOKEN: ${{ secrets.CLAWFIX_API_TOKEN }} + CLAWFIX_CANARY_TOKEN: ${{ secrets.CLAWFIX_CANARY_TOKEN }} + CLAWFIX_SCHEDULED_CANARY: ${{ vars.CLAWFIX_SCHEDULED_CANARY }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node 24 + uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Verify public production contract + shell: bash + run: | + set -euo pipefail + mode="none" + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + mode="${{ inputs.metered_mode }}" + elif [ -n "${CLAWFIX_SCHEDULED_CANARY:-}" ]; then + # Scheduled paid traffic is opt-in. No repository variable means free checks only. + mode="$CLAWFIX_SCHEDULED_CANARY" + fi + + case "$mode" in + none) canary_args=() ;; + agent) canary_args=(--agent-canary) ;; + diagnose) canary_args=(--diagnose-canary) ;; + *) echo "::error ::invalid canary mode: $mode"; exit 2 ;; + esac + + version="$(node -p "require('./package.json').version")" + node scripts/verify-production.mjs \ + --base-url https://clawfix.dev \ + --expected-version "$version" \ + "${canary_args[@]}" \ + 2>&1 | tee production-verification.json + + - name: Retain verification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: production-verification-${{ github.run_id }} + path: production-verification.json + if-no-files-found: warn + retention-days: 30 diff --git a/package.json b/package.json index c6af663..25f059a 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "build:tui": "node scripts/build-tui-release.mjs", "verify:tui": "node scripts/verify-tui-artifact.mjs", "smoke:tui": "node scripts/smoke-tui-binary.mjs", - "smoke:tui:interactive": "node scripts/smoke-tui-interactive.mjs" + "smoke:tui:interactive": "node scripts/smoke-tui-interactive.mjs", + "verify:production": "node scripts/verify-production.mjs" }, "dependencies": { "express": "^5.1.0", diff --git a/scripts/verify-production.mjs b/scripts/verify-production.mjs new file mode 100644 index 0000000..263d52d --- /dev/null +++ b/scripts/verify-production.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node + +import { createHash, randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const DEFAULT_BASE_URL = 'https://clawfix.dev'; +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_RESPONSE_BYTES = 1_000_000; +const PACKAGE_VERSION = JSON.parse( + readFileSync(new URL('../package.json', import.meta.url), 'utf8'), +).version; + +function invariant(value, message) { + if (!value) throw new Error(message); +} + +function safeMessage(error) { + const message = error instanceof Error ? error.message : String(error); + return message.replace(/[\r\n]+/g, ' ').slice(0, 500); +} + +function normalizeBaseUrl(value) { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error('base URL is invalid'); + } + invariant(!parsed.username && !parsed.password, 'base URL must not contain credentials'); + invariant(!parsed.search && !parsed.hash, 'base URL must not contain a query or fragment'); + invariant(parsed.pathname === '/', 'base URL must not contain a path'); + + const hostname = parsed.hostname.replace(/^\[|\]$/g, ''); + const isLoopback = hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1'; + invariant( + parsed.protocol === 'https:' || (parsed.protocol === 'http:' && isLoopback), + 'remote base URL must use HTTPS', + ); + return parsed.origin; +} + +async function readBoundedText(response, label) { + const advertisedLength = Number(response.headers?.get?.('content-length')); + if (Number.isFinite(advertisedLength) && advertisedLength > MAX_RESPONSE_BYTES) { + throw new Error(`${label} body exceeded ${MAX_RESPONSE_BYTES} bytes`); + } + + if (!response.body || typeof response.body.getReader !== 'function') { + invariant(typeof response.text === 'function', `${label} returned an invalid response`); + const text = await response.text(); + invariant( + Buffer.byteLength(text, 'utf8') <= MAX_RESPONSE_BYTES, + `${label} body exceeded ${MAX_RESPONSE_BYTES} bytes`, + ); + return text; + } + + const reader = response.body.getReader(); + const chunks = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = Buffer.from(value); + totalBytes += chunk.length; + if (totalBytes > MAX_RESPONSE_BYTES) { + try { + await reader.cancel(); + } catch { + // The size failure remains authoritative. + } + throw new Error(`${label} body exceeded ${MAX_RESPONSE_BYTES} bytes`); + } + chunks.push(chunk); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, totalBytes).toString('utf8'); +} + +async function requestText(fetchImpl, url, options, { label, timeoutMs }) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + let response; + try { + response = await fetchImpl(url, { + ...options, + redirect: 'error', + signal: controller.signal, + }); + } catch (error) { + if (controller.signal.aborted) { + throw new Error(`${label} timed out after ${timeoutMs}ms`); + } + throw new Error(`${label} request failed: ${safeMessage(error)}`); + } + invariant(response && typeof response === 'object', `${label} returned an invalid response`); + invariant(response.ok, `${label} returned HTTP ${response.status}`); + const text = await readBoundedText(response, label); + return { response, text }; + } finally { + clearTimeout(timer); + } +} + +function parseJson(text, label) { + try { + return JSON.parse(text); + } catch { + throw new Error(`${label} returned invalid JSON`); + } +} + +export function parseSseEvents(raw) { + const events = []; + const normalized = String(raw).replace(/\r\n/g, '\n'); + invariant(normalized.length === 0 || /\n\n+$/.test(normalized), 'unterminated SSE frame'); + const frames = normalized.split(/\n\n+/); + for (const frame of frames) { + if (!frame.trim()) continue; + let event = 'message'; + const data = []; + for (const line of frame.split('\n')) { + if (line.startsWith('event:')) event = line.slice(6).trim(); + else if (line.startsWith('data:')) data.push(line.slice(5).trimStart()); + } + if (data.length === 0) continue; + const payload = data.join('\n'); + try { + events.push({ event, data: JSON.parse(payload) }); + } catch { + throw new Error(`invalid SSE JSON for ${event}`); + } + } + return events; +} + +export function parseVerifierArgs(argv) { + const parsed = { + baseUrl: DEFAULT_BASE_URL, + expectedVersion: PACKAGE_VERSION, + meteredMode: 'none', + timeoutMs: DEFAULT_TIMEOUT_MS, + }; + let agent = false; + let diagnose = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--base-url') parsed.baseUrl = argv[++index]; + else if (arg === '--expected-version') parsed.expectedVersion = argv[++index]; + else if (arg === '--timeout-ms') parsed.timeoutMs = Number.parseInt(argv[++index], 10); + else if (arg === '--agent-canary') agent = true; + else if (arg === '--diagnose-canary') diagnose = true; + else if (arg === '--help' || arg === '-h') { + throw new Error( + 'usage: verify-production [--base-url URL] [--expected-version X.Y.Z] ' + + '[--timeout-ms N] [--agent-canary | --diagnose-canary]', + ); + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + invariant(typeof parsed.baseUrl === 'string' && parsed.baseUrl.length > 0, '--base-url requires a value'); + invariant( + typeof parsed.expectedVersion === 'string' && /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(parsed.expectedVersion), + '--expected-version requires a semantic version', + ); + invariant(Number.isSafeInteger(parsed.timeoutMs) && parsed.timeoutMs > 0, '--timeout-ms must be a positive integer'); + invariant(!(agent && diagnose), '--agent-canary and --diagnose-canary are mutually exclusive'); + if (agent) parsed.meteredMode = 'agent'; + if (diagnose) parsed.meteredMode = 'diagnose'; + return parsed; +} + +async function verifyFreeSurface({ baseUrl, expectedVersion, fetchImpl, timeoutMs }) { + const requestOptions = { label: '', timeoutMs }; + const [rootResult, healthResult, statsResult, installerResult, installerHashResult] = await Promise.all([ + requestText(fetchImpl, `${baseUrl}/`, { headers: { accept: 'text/html' } }, { + ...requestOptions, + label: 'root', + }), + requestText(fetchImpl, `${baseUrl}/api/health`, { headers: { accept: 'application/json' } }, { + ...requestOptions, + label: 'health', + }), + requestText(fetchImpl, `${baseUrl}/api/stats`, { headers: { accept: 'application/json' } }, { + ...requestOptions, + label: 'stats', + }), + requestText(fetchImpl, `${baseUrl}/install`, { headers: { accept: 'text/plain' } }, { + ...requestOptions, + label: 'installer', + }), + requestText(fetchImpl, `${baseUrl}/install/sha256`, { headers: { accept: 'application/json' } }, { + ...requestOptions, + label: 'installer hash', + }), + ]); + + invariant(/ClawFix/i.test(rootResult.text), 'root did not contain the ClawFix product marker'); + + const health = parseJson(healthResult.text, 'health'); + invariant(health?.status === 'ok', `health status was ${String(health?.status)}`); + invariant(Number.isFinite(Date.parse(health?.timestamp)), 'health timestamp was invalid'); + + const stats = parseJson(statsResult.text, 'stats'); + invariant( + stats?.version === expectedVersion, + `expected version ${expectedVersion}, received ${String(stats?.version)}`, + ); + invariant(Number.isFinite(Number(stats?.totalDiagnoses)), 'stats totalDiagnoses was invalid'); + + const hashDocument = parseJson(installerHashResult.text, 'installer hash'); + const advertisedHash = hashDocument?.sha256; + invariant(typeof advertisedHash === 'string' && /^[a-f0-9]{64}$/i.test(advertisedHash), 'installer hash was invalid'); + const calculatedHash = createHash('sha256').update(installerResult.text).digest('hex'); + invariant(calculatedHash === advertisedHash, 'installer SHA-256 mismatch'); + const headerHash = installerResult.response.headers?.get?.('x-script-sha256'); + invariant(headerHash === advertisedHash, 'installer response header SHA-256 mismatch'); + + return [ + { name: 'root', ok: true }, + { name: 'health', ok: true, status: health.status }, + { name: 'stats', ok: true, version: stats.version }, + { name: 'installer', ok: true, sha256: calculatedHash }, + ]; +} + +async function verifyAgentCanary({ baseUrl, fetchImpl, timeoutMs, apiToken }) { + const conversationId = randomUUID(); + const headers = { accept: 'text/event-stream', 'content-type': 'application/json' }; + if (apiToken) headers.authorization = `Bearer ${apiToken}`; + const result = await requestText(fetchImpl, `${baseUrl}/api/v2/agent/messages`, { + method: 'POST', + headers, + body: JSON.stringify({ + conversationId, + message: 'ClawFix continuity canary. Reply with one short sentence and do not propose a repair.', + availableRepairs: [], + }), + }, { label: 'agent canary', timeoutMs }); + invariant( + result.response.headers?.get?.('content-type')?.includes('text/event-stream'), + 'agent canary did not return SSE', + ); + const events = parseSseEvents(result.text); + invariant(!events.some((event) => event.event === 'agent.error'), 'agent canary emitted agent.error'); + const metaEvents = events.filter((event) => event.event === 'agent.meta'); + invariant(metaEvents.length === 1, 'agent canary must emit exactly one agent.meta event'); + const meta = metaEvents[0]; + invariant(events[0] === meta, 'agent.meta must be the first event'); + const deltas = events.filter((event) => event.event === 'assistant.delta'); + const doneEvents = events.filter((event) => event.event === 'agent.done'); + invariant(doneEvents.length === 1, 'agent canary must emit exactly one agent.done event'); + const done = doneEvents[0]; + invariant(events.at(-1) === done, 'agent.done must be the final event'); + invariant(meta?.data?.conversationId === conversationId, 'agent canary metadata did not match the conversation'); + const assistantText = deltas + .map((event) => typeof event.data?.text === 'string' ? event.data.text : '') + .join(''); + invariant(assistantText.trim().length > 0, 'agent canary returned no assistant text'); + invariant(done?.data?.conversationId === conversationId, 'agent canary did not complete'); + return { name: 'agent-canary', ok: true, sseEvents: events.length }; +} + +async function verifyDiagnosisCanary({ baseUrl, fetchImpl, timeoutMs, apiToken, canaryToken }) { + invariant(canaryToken, 'CLAWFIX_CANARY_TOKEN is required for --diagnose-canary'); + const headers = { + accept: 'application/json', + 'content-type': 'application/json', + 'x-clawfix-canary': canaryToken, + }; + if (apiToken) headers.authorization = `Bearer ${apiToken}`; + const result = await requestText(fetchImpl, `${baseUrl}/api/diagnose`, { + method: 'POST', + headers, + body: JSON.stringify({ + system: { os: 'clawfix-continuity-canary', arch: 'x64' }, + openclaw: { version: 'canary', processExists: true, portListening: true }, + service: { manager: 'synthetic', state: 'running', exitCode: 0 }, + logs: { errors: '', sigtermCount: 0, errLogSizeMB: 0 }, + _localIssues: [], + }), + }, { label: 'diagnosis canary', timeoutMs }); + const diagnostic = parseJson(result.text, 'diagnosis canary'); + invariant(typeof diagnostic?.fixId === 'string' && diagnostic.fixId.length >= 10, 'diagnosis canary returned no fix ID'); + invariant(typeof diagnostic?.analysis === 'string' && diagnostic.analysis.length > 0, 'diagnosis canary returned no analysis'); + invariant( + diagnostic.canary === true && diagnostic.persisted === true, + 'diagnosis canary classification or persistence was not acknowledged', + ); + return { name: 'diagnosis-canary', ok: true, fixIdPresent: true }; +} + +export async function runProductionVerification({ + baseUrl = DEFAULT_BASE_URL, + expectedVersion = PACKAGE_VERSION, + meteredMode = 'none', + timeoutMs = DEFAULT_TIMEOUT_MS, + apiToken = '', + canaryToken = '', + fetchImpl = globalThis.fetch, +} = {}) { + invariant(typeof fetchImpl === 'function', 'fetch is unavailable'); + invariant(['none', 'agent', 'diagnose'].includes(meteredMode), `invalid metered mode: ${meteredMode}`); + if (meteredMode === 'diagnose') { + invariant(canaryToken, 'CLAWFIX_CANARY_TOKEN is required for --diagnose-canary'); + } + + const normalizedBaseUrl = normalizeBaseUrl(baseUrl); + const startedAt = new Date().toISOString(); + const checks = await verifyFreeSurface({ + baseUrl: normalizedBaseUrl, + expectedVersion, + fetchImpl, + timeoutMs, + }); + + if (meteredMode === 'agent') { + checks.push(await verifyAgentCanary({ + baseUrl: normalizedBaseUrl, + fetchImpl, + timeoutMs, + apiToken, + })); + } else if (meteredMode === 'diagnose') { + checks.push(await verifyDiagnosisCanary({ + baseUrl: normalizedBaseUrl, + fetchImpl, + timeoutMs, + apiToken, + canaryToken, + })); + } + + return { + ok: true, + baseUrl: normalizedBaseUrl, + expectedVersion, + meteredMode, + startedAt, + finishedAt: new Date().toISOString(), + checks, + }; +} + +async function main() { + try { + const options = parseVerifierArgs(process.argv.slice(2)); + const report = await runProductionVerification({ + ...options, + apiToken: process.env.CLAWFIX_API_TOKEN || '', + canaryToken: process.env.CLAWFIX_CANARY_TOKEN || '', + }); + console.log(JSON.stringify(report, null, 2)); + } catch (error) { + console.error(JSON.stringify({ + ok: false, + error: safeMessage(error), + finishedAt: new Date().toISOString(), + }, null, 2)); + process.exitCode = 1; + } +} + +const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : ''; +if (import.meta.url === invokedPath) await main(); diff --git a/src/db.js b/src/db.js index a9d0934..7e4d65c 100644 --- a/src/db.js +++ b/src/db.js @@ -4,6 +4,25 @@ const { Pool } = pg; let pool = null; +const PUBLIC_DIAGNOSIS_FILTER = "source IS DISTINCT FROM 'canary'"; + +export function shouldCountDiagnosisInPublicMetrics(source) { + return source !== 'canary'; +} + +/** Public dashboard queries over diagnoses. Canary rows stay available for operations only. */ +export function getPublicStatsQueries() { + return Object.freeze({ + total: `SELECT COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER}`, + today: `SELECT COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} AND created_at > NOW() - INTERVAL '24 hours'`, + versions: `SELECT openclaw_version, COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} AND openclaw_version IS NOT NULL GROUP BY openclaw_version ORDER BY count DESC LIMIT 5`, + outcomes: `SELECT outcome, COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} GROUP BY outcome`, + serviceManagers: `SELECT service_manager, COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} AND service_manager IS NOT NULL GROUP BY service_manager ORDER BY count DESC`, + sigterms: `SELECT COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} AND (sigterm_count > 0 OR service_state = 'sigterm')`, + zombies: `SELECT COUNT(*) as count FROM diagnoses WHERE ${PUBLIC_DIAGNOSIS_FILTER} AND (service_state = 'crashed' OR service_state = 'failed')`, + }); +} + export function getPool() { if (!pool && process.env.DATABASE_URL) { pool = new Pool({ @@ -118,7 +137,7 @@ export async function initDB() { */ export async function storeDiagnosis(result, source = 'cli') { const db = getPool(); - if (!db) return; + if (!db) return false; try { await db.query(` @@ -150,7 +169,7 @@ export async function storeDiagnosis(result, source = 'cli') { ]); // Update pattern detection counts - if (result.knownIssues) { + if (shouldCountDiagnosisInPublicMetrics(source) && result.knownIssues) { for (const issue of result.knownIssues) { await db.query(` INSERT INTO patterns (id, title, severity, times_detected, last_seen) @@ -161,8 +180,10 @@ export async function storeDiagnosis(result, source = 'cli') { `, [issue.id, issue.title, issue.severity]); } } + return true; } catch (err) { console.error('Store diagnosis failed:', err.message); + return false; } } @@ -186,8 +207,8 @@ export async function storeFeedback(fixId, success, issuesRemaining, comment) { // Update pattern success rates if (success) { - const diag = await db.query('SELECT issues_pattern FROM diagnoses WHERE id = $1', [fixId]); - if (diag.rows[0]) { + const diag = await db.query('SELECT issues_pattern, source FROM diagnoses WHERE id = $1', [fixId]); + if (diag.rows[0] && shouldCountDiagnosisInPublicMetrics(diag.rows[0].source)) { const patterns = diag.rows[0].issues_pattern || []; for (const patternId of patterns) { await db.query(` @@ -207,8 +228,7 @@ export async function storeFeedback(fixId, success, issuesRemaining, comment) { /** * Retrieve a diagnosis by fix ID (for results page persistence) */ -export async function getDiagnosis(fixId) { - const db = getPool(); +export async function getDiagnosis(fixId, db = getPool()) { if (!db) return null; try { @@ -235,6 +255,7 @@ export async function getDiagnosis(fixId) { return { fixId: row.id, + _source: row.source || 'unknown', timestamp: row.created_at.toISOString(), issuesFound: row.issues_count, knownIssues, @@ -264,15 +285,16 @@ export async function getStats() { if (!db) return null; try { + const queries = getPublicStatsQueries(); const [total, today, topIssues, versions, outcomes, serviceManagers, sigterms, zombies] = await Promise.all([ - db.query('SELECT COUNT(*) as count FROM diagnoses'), - db.query("SELECT COUNT(*) as count FROM diagnoses WHERE created_at > NOW() - INTERVAL '24 hours'"), + db.query(queries.total), + db.query(queries.today), db.query('SELECT id, title, severity, times_detected, success_rate FROM patterns ORDER BY times_detected DESC LIMIT 10'), - db.query('SELECT openclaw_version, COUNT(*) as count FROM diagnoses WHERE openclaw_version IS NOT NULL GROUP BY openclaw_version ORDER BY count DESC LIMIT 5'), - db.query("SELECT outcome, COUNT(*) as count FROM diagnoses GROUP BY outcome"), - db.query("SELECT service_manager, COUNT(*) as count FROM diagnoses WHERE service_manager IS NOT NULL GROUP BY service_manager ORDER BY count DESC"), - db.query("SELECT COUNT(*) as count FROM diagnoses WHERE sigterm_count > 0 OR service_state = 'sigterm'"), - db.query("SELECT COUNT(*) as count FROM diagnoses WHERE service_state = 'crashed' OR service_state = 'failed'"), + db.query(queries.versions), + db.query(queries.outcomes), + db.query(queries.serviceManagers), + db.query(queries.sigterms), + db.query(queries.zombies), ]); return { diff --git a/src/routes/diagnose.js b/src/routes/diagnose.js index 968daac..b5c6d50 100644 --- a/src/routes/diagnose.js +++ b/src/routes/diagnose.js @@ -14,6 +14,7 @@ import { redactOutbound, validateFixId } from '../../cli/bin/security.js'; import { clientIp, createRateLimiter, + isAuthorizedCanaryRequest, isPaidAIEnabled, positiveEnvInteger, sharedAIRequestGuard, @@ -32,6 +33,11 @@ const diagnoseLimiter = createRateLimiter({ windowMs: positiveEnvInteger(process.env.RATE_LIMIT_WINDOW_MS, 60_000), }); +export function diagnosisSource(req, env = process.env) { + if (isAuthorizedCanaryRequest(req, env)) return 'canary'; + return req?.headers?.['user-agent']?.includes('node') ? 'npx' : 'curl'; +} + const SYSTEM_PROMPT = `You are ClawFix, an expert AI diagnostician for OpenClaw installations. You analyze redacted diagnostic data from users' OpenClaw setups and provide advisory findings. @@ -128,6 +134,7 @@ diagnoseRouter.post('/diagnose', async (req, res) => { // Redact again at the service boundary before AI, persistence, or response. const diagnostic = redactOutbound(req.body); + const source = diagnosisSource(req); // Step 1: Pattern matching (fast, free) let knownIssues = detectIssues(diagnostic); @@ -198,13 +205,22 @@ diagnoseRouter.post('/diagnose', async (req, res) => { _processExists: diagnostic.openclaw?.processExists ?? null, _portListening: diagnostic.openclaw?.portListening ?? null, _aiIssues: aiAnalysis.additionalIssues || [], + _source: source, }); fixes.set(fixId, result); // Persist to database - const source = req.headers['user-agent']?.includes('node') ? 'npx' : 'curl'; - storeDiagnosis(result, source).catch(() => {}); + const persistence = storeDiagnosis(result, source); + if (source === 'canary') { + const persisted = await persistence; + if (!persisted) { + fixes.delete(fixId); + return res.status(503).json({ error: 'Canary persistence failed' }); + } + } else { + persistence.catch(() => {}); + } // Clean up old fixes (keep last 1000) if (fixes.size > 1000) { @@ -213,8 +229,10 @@ diagnoseRouter.post('/diagnose', async (req, res) => { } // Strip internal metadata before sending to client - const { _hostHash, _os, _arch, _nodeVersion, _openclawVersion, _serviceManager, _serviceState, _serviceExitCode, _errLogSizeMB, _sigtermCount, _processExists, _portListening, _aiIssues, ...clientResult } = result; - res.json(clientResult); + const { _hostHash, _os, _arch, _nodeVersion, _openclawVersion, _serviceManager, _serviceState, _serviceExitCode, _errLogSizeMB, _sigtermCount, _processExists, _portListening, _aiIssues, _source, ...clientResult } = result; + res.json(source === 'canary' + ? { ...clientResult, canary: true, persisted: true } + : clientResult); } catch (error) { console.error('Diagnosis error:', redactOutbound(error?.message || 'unknown error')); res.status(500).json({ error: 'Diagnosis failed' }); @@ -251,16 +269,19 @@ diagnoseRouter.get('/fix/:fixId', async (req, res) => { } // Strip internal metadata - const { _hostHash, _os, _arch, _nodeVersion, _openclawVersion, _serviceManager, _serviceState, _serviceExitCode, _errLogSizeMB, _sigtermCount, _processExists, _portListening, _aiIssues, ...clientFix } = fix; + const { _hostHash, _os, _arch, _nodeVersion, _openclawVersion, _serviceManager, _serviceState, _serviceExitCode, _errLogSizeMB, _sigtermCount, _processExists, _portListening, _aiIssues, _source, ...clientFix } = fix; res.json(clientFix); }); // Stats endpoint diagnoseRouter.get('/stats', async (req, res) => { const dbStats = await getStats(); + const inMemoryPublicCount = [...fixes.values()] + .filter(fix => fix?._source !== 'canary') + .length; res.json({ - totalDiagnoses: dbStats?.totalDiagnoses || fixes.size, + totalDiagnoses: dbStats?.totalDiagnoses ?? inMemoryPublicCount, last24h: dbStats?.last24h || 0, topIssues: dbStats?.topIssues || [], versionBreakdown: dbStats?.versionBreakdown || [], diff --git a/src/security.js b/src/security.js index 2f391a3..eacfa7b 100644 --- a/src/security.js +++ b/src/security.js @@ -93,6 +93,18 @@ function tokensEqual(expected, received) { return left.length === right.length && timingSafeEqual(left, right); } +/** + * Recognize an operational canary without turning its marker into an auth bypass. + * Missing configuration, non-string headers, and any mismatch all fail closed. + */ +export function isAuthorizedCanaryRequest(req, env = process.env) { + const expected = env?.CLAWFIX_CANARY_TOKEN; + const received = req?.headers?.['x-clawfix-canary']; + if (typeof expected !== 'string' || expected.length === 0) return false; + if (typeof received !== 'string' || received.length === 0) return false; + return tokensEqual(expected, received); +} + export function createAIRequestGuard({ token = '', dailyLimit = 200, diff --git a/test/canary-metrics.test.js b/test/canary-metrics.test.js new file mode 100644 index 0000000..c6c6225 --- /dev/null +++ b/test/canary-metrics.test.js @@ -0,0 +1,110 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + createAIRequestGuard, + isAuthorizedCanaryRequest, +} from '../src/security.js'; +import { + getDiagnosis, + getPublicStatsQueries, + shouldCountDiagnosisInPublicMetrics, + storeDiagnosis, +} from '../src/db.js'; +import { diagnosisSource } from '../src/routes/diagnose.js'; + +function request(headers = {}) { + return { headers, ip: '127.0.0.1', socket: { remoteAddress: '127.0.0.1' } }; +} + +test('canary recognition requires a configured exact token and uses fail-closed comparisons', () => { + const env = { CLAWFIX_CANARY_TOKEN: 'correct-token' }; + assert.equal( + isAuthorizedCanaryRequest(request({ 'x-clawfix-canary': 'correct-token' }), env), + true, + ); + assert.equal(isAuthorizedCanaryRequest(request(), env), false); + assert.equal( + isAuthorizedCanaryRequest(request({ 'x-clawfix-canary': 'wrong-token' }), env), + false, + ); + assert.equal( + isAuthorizedCanaryRequest(request({ 'x-clawfix-canary': 'x' }), env), + false, + ); + assert.equal( + isAuthorizedCanaryRequest(request({ 'x-clawfix-canary': ['correct-token'] }), env), + false, + ); + assert.equal( + isAuthorizedCanaryRequest(request({ 'x-clawfix-canary': 'correct-token' }), {}), + false, + ); +}); + +test('canary marker classifies storage source but cannot bypass AI bearer authorization', () => { + const req = request({ 'x-clawfix-canary': 'canary-token', 'user-agent': 'node/test' }); + assert.equal(diagnosisSource(req, { CLAWFIX_CANARY_TOKEN: 'canary-token' }), 'canary'); + assert.equal(diagnosisSource(req, { CLAWFIX_CANARY_TOKEN: 'different' }), 'npx'); + + const guard = createAIRequestGuard({ token: 'api-token', dailyLimit: 1, concurrency: 1 }); + const denied = guard.acquire(req); + assert.deepEqual(denied, { allowed: false, status: 401, error: 'Unauthorized' }); +}); + +test('only authorized non-canary diagnoses contribute to public aggregate state', () => { + assert.equal(shouldCountDiagnosisInPublicMetrics('canary'), false); + assert.equal(shouldCountDiagnosisInPublicMetrics('curl'), true); + assert.equal(shouldCountDiagnosisInPublicMetrics('npx'), true); + assert.equal(shouldCountDiagnosisInPublicMetrics(undefined), true); +}); + +test('every diagnosis-backed public stats query excludes canary rows', () => { + const queries = getPublicStatsQueries(); + for (const [name, sql] of Object.entries(queries)) { + assert.match(sql, /\bdiagnoses\b/, name); + assert.match(sql, /source IS DISTINCT FROM 'canary'/, name); + } +}); + +test('canary persistence reports failure when durable storage is unavailable', async () => { + const previous = process.env.DATABASE_URL; + delete process.env.DATABASE_URL; + try { + assert.equal(await storeDiagnosis({ fixId: 'canaryFix12' }, 'canary'), false); + } finally { + if (previous === undefined) delete process.env.DATABASE_URL; + else process.env.DATABASE_URL = previous; + } +}); + +test('rehydrated canaries retain their source and stay out of fallback memory counts', async () => { + const row = { + id: 'canaryRehydrated12', + created_at: new Date('2026-08-01T00:00:00.000Z'), + issues_count: 0, + known_issues_detail: [], + issues_pattern: [], + ai_summary: 'canary', + source: 'canary', + }; + const db = { + async query(sql) { + assert.match(sql, /FROM diagnoses/); + return { rows: [row] }; + }, + }; + const previous = process.env.DATABASE_URL; + delete process.env.DATABASE_URL; + try { + const rehydrated = await getDiagnosis(row.id, db); + assert.equal(rehydrated?._source, 'canary'); + const fallbackPublicCount = [rehydrated] + .filter((fix) => fix?._source !== 'canary') + .length; + assert.equal(fallbackPublicCount, 0); + } finally { + if (previous === undefined) delete process.env.DATABASE_URL; + else process.env.DATABASE_URL = previous; + } +}); diff --git a/test/production-continuity.test.js b/test/production-continuity.test.js new file mode 100644 index 0000000..9b15ee2 --- /dev/null +++ b/test/production-continuity.test.js @@ -0,0 +1,554 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + parseSseEvents, + parseVerifierArgs, + runProductionVerification, +} from '../scripts/verify-production.mjs'; + +const INSTALLER = '#!/bin/sh\nprintf "ClawFix installer\\n"\n'; +const INSTALLER_HASH = createHash('sha256').update(INSTALLER).digest('hex'); +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); + +function response(body, { status = 200, headers = {} } = {}) { + return new Response(body, { status, headers }); +} + +function verifierFetch({ + version = '0.11.2', + installerHash = INSTALLER_HASH, + rootBody = 'ClawFix

ClawFix

', + handler, +} = {}) { + const calls = []; + const fetchImpl = async (url, options = {}) => { + const parsed = new URL(url); + calls.push({ path: parsed.pathname, options }); + if (handler) { + const handled = await handler(parsed, options); + if (handled) return handled; + } + switch (parsed.pathname) { + case '/': + return response(rootBody, { + headers: { 'content-type': 'text/html' }, + }); + case '/api/health': + return response(JSON.stringify({ status: 'ok', timestamp: '2026-08-01T00:00:00.000Z' }), { + headers: { 'content-type': 'application/json' }, + }); + case '/api/stats': + return response(JSON.stringify({ version, totalDiagnoses: 200 }), { + headers: { 'content-type': 'application/json' }, + }); + case '/install': + return response(INSTALLER, { + headers: { + 'content-type': 'text/plain', + 'x-script-sha256': INSTALLER_HASH, + }, + }); + case '/install/sha256': + return response(JSON.stringify({ sha256: installerHash }), { + headers: { 'content-type': 'application/json' }, + }); + default: + return response('not found', { status: 404 }); + } + }; + return { fetchImpl, calls }; +} + +test('free production verification checks root, health, stats, installer, and exact hash', async () => { + const { fetchImpl, calls } = verifierFetch(); + const report = await runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + fetchImpl, + timeoutMs: 1_000, + }); + + assert.equal(report.ok, true); + assert.equal(report.expectedVersion, '0.11.2'); + assert.deepEqual( + report.checks.map((check) => check.name).sort(), + ['health', 'installer', 'root', 'stats'].sort(), + ); + assert.deepEqual( + calls.map((call) => call.path).sort(), + ['/', '/api/health', '/api/stats', '/install', '/install/sha256'].sort(), + ); + assert.ok(calls.every((call) => call.options.redirect === 'error')); +}); + +test('production verification fails closed on version drift and installer hash mismatch', async () => { + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.12.0', + fetchImpl: verifierFetch({ version: '0.11.2' }).fetchImpl, + }), + /expected version 0\.12\.0, received 0\.11\.2/, + ); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + fetchImpl: verifierFetch({ installerHash: '0'.repeat(64) }).fetchImpl, + }), + /installer SHA-256 mismatch/, + ); +}); + +test('remote verification refuses insecure or credential-bearing base URLs before fetching', async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + throw new Error('must not fetch'); + }; + + await assert.rejects( + runProductionVerification({ + baseUrl: 'http://clawfix.dev', + expectedVersion: '0.11.2', + fetchImpl, + }), + /must use https/i, + ); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://user:password@clawfix.dev', + expectedVersion: '0.11.2', + fetchImpl, + }), + /must not contain credentials/i, + ); + for (const baseUrl of [ + 'https://clawfix.dev/api', + 'https://clawfix.dev/?source=test', + 'https://clawfix.dev/#fragment', + ]) { + await assert.rejects( + runProductionVerification({ baseUrl, expectedVersion: '0.11.2', fetchImpl }), + /must not contain (?:a path|a query or fragment)/i, + ); + } + assert.equal(calls, 0); +}); + +test('metered canaries never run after a free prerequisite drifts', async () => { + const { fetchImpl, calls } = verifierFetch({ version: '0.11.2' }); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.12.0', + meteredMode: 'agent', + apiToken: 'must-not-be-sent', + fetchImpl, + }), + /expected version 0\.12\.0, received 0\.11\.2/, + ); + assert.equal(calls.some((call) => call.path === '/api/v2/agent/messages'), false); +}); + +test('response bodies are rejected while reading once they exceed the verifier limit', async () => { + const oversized = verifierFetch({ + rootBody: 'C'.repeat((1024 * 1024) + 1), + }); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + fetchImpl: oversized.fetchImpl, + }), + /body exceeded/i, + ); +}); + +test('production verification rejects malformed health JSON and bounded request timeouts', async () => { + const malformed = verifierFetch({ + handler(parsed) { + if (parsed.pathname === '/api/health') { + return response('{nope', { headers: { 'content-type': 'application/json' } }); + } + return null; + }, + }); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + fetchImpl: malformed.fetchImpl, + }), + /health returned invalid JSON/, + ); + + const never = async (_url, { signal }) => new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + fetchImpl: never, + timeoutMs: 10, + }), + /timed out/, + ); +}); + +test('SSE parser requires complete event frames and rejects malformed JSON', () => { + const events = parseSseEvents([ + 'event: agent.meta', + 'data: {"conversationId":"abc"}', + '', + 'event: assistant.delta', + 'data: {"text":"ok"}', + '', + 'event: agent.done', + 'data: {"repairProposed":false}', + '', + '', + ].join('\n')); + assert.deepEqual(events.map((event) => event.event), [ + 'agent.meta', + 'assistant.delta', + 'agent.done', + ]); + assert.throws(() => parseSseEvents('event: agent.done\ndata: {bad}\n\n'), /invalid SSE JSON/); + assert.throws( + () => parseSseEvents('event: agent.done\ndata: {"repairProposed":false}\n'), + /unterminated SSE frame/, + ); +}); + +test('agent canary makes exactly one bounded metered request and validates SSE completion', async () => { + const { fetchImpl, calls } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + assert.equal(options.method, 'POST'); + assert.equal(options.headers.authorization, 'Bearer api-test-token'); + const body = JSON.parse(options.body); + assert.match(body.conversationId, /^[0-9a-f-]{36}$/); + assert.equal(body.availableRepairs.length, 0); + return response([ + 'event: agent.meta', + `data: {"conversationId":"${body.conversationId}"}`, + '', + 'event: assistant.delta', + 'data: {"text":"healthy"}', + '', + 'event: agent.done', + `data: {"conversationId":"${body.conversationId}","repairProposed":false}`, + '', + '', + ].join('\n'), { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + const report = await runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + apiToken: 'api-test-token', + fetchImpl, + }); + assert.equal(report.ok, true); + assert.equal(report.meteredMode, 'agent'); + assert.equal(calls.filter((call) => call.path === '/api/v2/agent/messages').length, 1); +}); + +test('agent canary rejects whitespace-only assistant output', async () => { + const { fetchImpl } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + const { conversationId } = JSON.parse(options.body); + return response([ + 'event: agent.meta', + `data: {"conversationId":"${conversationId}"}`, + '', + 'event: assistant.delta', + 'data: {"text":" \\n\\t"}', + '', + 'event: agent.done', + `data: {"conversationId":"${conversationId}","repairProposed":false}`, + '', + '', + ].join('\n'), { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + fetchImpl, + }), + /agent canary returned no assistant text/, + ); +}); + +test('agent canary rejects duplicate or mismatched protocol terminal events', async (t) => { + const scenarios = [ + { + name: 'duplicate agent.meta', + expected: /exactly one agent\.meta event/, + events(conversationId) { + return [ + ['agent.meta', { conversationId }], + ['agent.meta', { conversationId }], + ['assistant.delta', { text: 'healthy' }], + ['agent.done', { conversationId, repairProposed: false }], + ]; + }, + }, + { + name: 'duplicate agent.done', + expected: /exactly one agent\.done event/, + events(conversationId) { + return [ + ['agent.meta', { conversationId }], + ['assistant.delta', { text: 'healthy' }], + ['agent.done', { conversationId, repairProposed: false }], + ['agent.done', { conversationId, repairProposed: false }], + ]; + }, + }, + { + name: 'mismatched agent.done conversation', + expected: /agent canary did not complete/, + events(conversationId) { + return [ + ['agent.meta', { conversationId }], + ['assistant.delta', { text: 'healthy' }], + ['agent.done', { conversationId: 'different-conversation', repairProposed: false }], + ]; + }, + }, + ]; + + for (const scenario of scenarios) { + await t.test(scenario.name, async () => { + const { fetchImpl } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + const { conversationId } = JSON.parse(options.body); + const stream = scenario.events(conversationId) + .flatMap(([event, data]) => [`event: ${event}`, `data: ${JSON.stringify(data)}`, '']) + .concat('') + .join('\n'); + return response(stream, { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + fetchImpl, + }), + scenario.expected, + ); + }); + } +}); + +test('agent canary rejects agent.error even when partial text and agent.done are present', async () => { + const { fetchImpl } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + const { conversationId } = JSON.parse(options.body); + return response([ + 'event: agent.meta', + `data: {"conversationId":"${conversationId}"}`, + '', + 'event: assistant.delta', + 'data: {"text":"partial"}', + '', + 'event: agent.error', + 'data: {"error":"provider failed","fatal":true}', + '', + 'event: agent.done', + `data: {"conversationId":"${conversationId}","repairProposed":false}`, + '', + '', + ].join('\n'), { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + fetchImpl, + }), + /agent canary emitted agent\.error/, + ); +}); + +test('agent canary requires agent.done to be the final event', async () => { + const { fetchImpl } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + const { conversationId } = JSON.parse(options.body); + return response([ + 'event: agent.meta', + `data: {"conversationId":"${conversationId}"}`, + '', + 'event: assistant.delta', + 'data: {"text":"healthy"}', + '', + 'event: agent.done', + `data: {"conversationId":"${conversationId}","repairProposed":false}`, + '', + 'event: assistant.delta', + 'data: {"text":"late"}', + '', + '', + ].join('\n'), { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + fetchImpl, + }), + /agent\.done must be the final event/, + ); +}); + +test('agent canary requires agent.meta before assistant output', async () => { + const { fetchImpl } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/v2/agent/messages') return null; + const { conversationId } = JSON.parse(options.body); + return response([ + 'event: assistant.delta', + 'data: {"text":"early"}', + '', + 'event: agent.meta', + `data: {"conversationId":"${conversationId}"}`, + '', + 'event: agent.done', + `data: {"conversationId":"${conversationId}","repairProposed":false}`, + '', + '', + ].join('\n'), { headers: { 'content-type': 'text/event-stream' } }); + }, + }); + + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'agent', + fetchImpl, + }), + /agent\.meta must be the first event/, + ); +}); + +test('diagnosis canary requires its own token and never includes it in the body', async () => { + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'diagnose', + fetchImpl: verifierFetch().fetchImpl, + }), + /CLAWFIX_CANARY_TOKEN is required/, + ); + + const canaryToken = `secret-${randomUUID()}`; + const { fetchImpl, calls } = verifierFetch({ + handler(parsed, options) { + if (parsed.pathname !== '/api/diagnose') return null; + assert.equal(options.headers['x-clawfix-canary'], canaryToken); + assert.equal(options.body.includes(canaryToken), false); + return response(JSON.stringify({ + fixId: 'canaryFix12', + analysis: 'bounded canary', + canary: true, + persisted: true, + }), { + headers: { 'content-type': 'application/json' }, + }); + }, + }); + const report = await runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'diagnose', + canaryToken, + fetchImpl, + }); + assert.equal(report.ok, true); + assert.equal(calls.filter((call) => call.path === '/api/diagnose').length, 1); + assert.equal(JSON.stringify(report).includes(canaryToken), false); +}); + +test('diagnosis canary fails unless production acknowledges canary classification and persistence', async () => { + const canaryToken = `secret-${randomUUID()}`; + for (const diagnostic of [ + { fixId: 'canaryFix12', analysis: 'bounded canary' }, + { fixId: 'canaryFix12', analysis: 'bounded canary', canary: true, persisted: false }, + ]) { + const { fetchImpl } = verifierFetch({ + handler(parsed) { + if (parsed.pathname !== '/api/diagnose') return null; + return response(JSON.stringify(diagnostic), { + headers: { 'content-type': 'application/json' }, + }); + }, + }); + await assert.rejects( + runProductionVerification({ + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'diagnose', + canaryToken, + fetchImpl, + }), + /classification or persistence was not acknowledged/i, + ); + } +}); + +test('CLI parsing caps each invocation at one explicit metered mode', () => { + assert.deepEqual( + parseVerifierArgs(['--base-url', 'https://example.test', '--expected-version', '0.11.2']), + { + baseUrl: 'https://example.test', + expectedVersion: '0.11.2', + meteredMode: 'none', + timeoutMs: 15_000, + }, + ); + assert.throws( + () => parseVerifierArgs(['--agent-canary', '--diagnose-canary']), + /mutually exclusive/, + ); + assert.throws(() => parseVerifierArgs(['--timeout-ms', '0']), /positive integer/); +}); + +test('scheduled workflow is free by default and retains machine-readable evidence', async () => { + const workflow = await readFile(join(ROOT, '.github/workflows/production-smoke.yml'), 'utf8'); + assert.match(workflow, /schedule:/); + assert.match(workflow, /workflow_dispatch:/); + assert.match(workflow, /CLAWFIX_SCHEDULED_CANARY: \$\{\{ vars\.CLAWFIX_SCHEDULED_CANARY \}\}/); + assert.match(workflow, /mode="none"/); + assert.match(workflow, /production-verification\.json/); + assert.match(workflow, /actions\/upload-artifact@v4/); +});