From 8f45531657dfff9a45fdb49e1a4ece76c4df9478 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Thu, 16 Jul 2026 09:00:59 +0400 Subject: [PATCH 1/3] fix: keep Codex MCP responsive during startup --- README.md | 4 +- docs/auto-refresh.md | 4 +- docs/reference/cli-and-mcp.md | 4 +- docs/release.md | 2 +- docs/tutorials/agent-quickstarts.md | 1 + src/infrastructure/background-auto-refresh.ts | 240 ++++++++++++++++++ src/infrastructure/install.ts | 2 + src/infrastructure/watch.ts | 4 + src/runtime/stdio-server.ts | 96 ++++--- src/runtime/stdio/prompts.ts | 9 +- src/runtime/stdio/resources.ts | 10 +- tests/unit/background-auto-refresh.test.ts | 217 ++++++++++++++++ tests/unit/install.test.ts | 20 +- 13 files changed, 575 insertions(+), 38 deletions(-) create mode 100644 src/infrastructure/background-auto-refresh.ts create mode 100644 tests/unit/background-auto-refresh.test.ts diff --git a/README.md b/README.md index 6f603c43..620e2187 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,9 @@ Madar supports these project-local installers: Installer details are in the [CLI and MCP reference](https://github.com/mohanagy/madar/blob/main/docs/reference/cli-and-mcp.md). Step-by-step setup and smoke tests are in the [agent quickstarts](https://github.com/mohanagy/madar/blob/main/docs/tutorials/agent-quickstarts.md). -If you upgrade from a version earlier than `0.30.0`, rerun your agent's install command to add automatic refresh to its managed MCP entry. +After upgrading Madar, rerun your agent's install command so its managed profile receives current runtime settings. Older profiles may lack automatic refresh; older Codex profiles may also lack the extended MCP startup window needed by large or synchronized workspaces. + +Codex installs set `startup_timeout_sec = 180`. Madar makes the MCP transport available while the initial graph reconciliation runs in a background worker, but graph-backed calls remain safely unavailable until `madar status` reports the watcher as `idle`. ## What Changes for the Agent diff --git a/docs/auto-refresh.md b/docs/auto-refresh.md index 4866607b..6e87003d 100644 --- a/docs/auto-refresh.md +++ b/docs/auto-refresh.md @@ -1,6 +1,6 @@ # Auto-refresh and generation policy -Installed MCP profiles run `madar serve --stdio --auto-refresh`. The server starts a recursive filesystem listener before its initial graph reconciliation, marks the graph pending as soon as a relevant event arrives, and performs an authoritative source snapshot before publishing the graph as usable again. +Installed MCP profiles run `madar serve --stdio --auto-refresh`. The stdio transport becomes available immediately while automatic refresh runs in a background worker. Before that worker starts, Madar publishes a `starting` watcher state so graph-backed requests fail closed instead of reading an older graph. The worker then starts a recursive filesystem listener before its initial graph reconciliation, marks the graph pending as soon as a relevant event arrives, and performs an authoritative source snapshot before publishing the graph as usable again. Filesystem events provide low-latency invalidation; they are not the correctness boundary. Madar also performs full reconciliations on an adaptive schedule. Idle intervals back off from 30 seconds to at most 5 minutes when recursive events are available. Platforms without recursive events use adaptive polling from 1 second to at most 30 seconds. The lower-level `pollIntervalMs` option is an internal/test override rather than a CLI setting. @@ -38,6 +38,8 @@ The local `watcher-state.json` beside `graph.json` is written atomically and inc `madar doctor` and `madar status` render those fields. During an auto-refresh MCP session, graph-backed prompts, resources, completions, and tool calls are refused while state is pending, reconciling, failed, incomplete, or policy-mismatched. Retry after the state returns to `idle`; if it remains failed, run `madar generate . --update` and inspect `madar status`. +MCP initialization, ping, and list/discovery requests remain responsive during `starting` and `reconciling`. This lets an agent connect without waiting for a cold large-repository build while preserving the same freshness boundary for every graph answer. + The refresh lease serializes multiple MCP processes that target the same workspace. Graph, source-manifest, indexing-manifest, report, and watcher-state publications use same-filesystem atomic renames. A post-build reconciliation detects edits made while generation was running and queues another rebuild before the state can return to `idle`. ## Linked worktrees diff --git a/docs/reference/cli-and-mcp.md b/docs/reference/cli-and-mcp.md index 5ffab098..ff8b11a5 100644 --- a/docs/reference/cli-and-mcp.md +++ b/docs/reference/cli-and-mcp.md @@ -33,7 +33,7 @@ For Claude, Cursor, Copilot, and Gemini, `--profile strict` writes `MADAR_TOOL_P Aider and OpenCode are intentionally context-pack-first: run `madar generate .`, install the profile, and start broad codebase work with `madar pack "" --task explain` before raw file search. `madar aider install` writes an AGENTS.md profile only; remove it with `madar aider uninstall`. `madar opencode install` writes the AGENTS.md profile, `.opencode/plugins/madar.js`, and a strict-profile Madar MCP entry in `opencode.json` or `opencode.jsonc`; remove only Madar-owned content with `madar opencode uninstall`. -Codex is intentionally context-pack-first too: run `madar generate .`, install with `madar codex install`, and start broad codebase work with `madar pack "" --task explain` before raw file search. The install writes the Madar-owned AGENTS.md section, `.codex/hooks.json`, `.codex/madar-user-prompt-submit.cjs`, and a marker-owned strict-profile `[mcp_servers.madar]` block in `.codex/config.toml`. Its `UserPromptSubmit` hook provides model-visible guidance only for local code tasks; it is guidance, not enforcement. Enable it only in a trusted repository, restart or start a new Codex session, use `/hooks` to review and trust the project hook, then verify the server through `/mcp` or `codex mcp list`. `madar doctor` and `madar status` validate on-disk install state only, not live Codex trust or activation. To remove the profile, run `madar codex uninstall`; it removes only Madar-owned AGENTS, hook, script, and marked TOML content while preserving unrelated content. +Codex is intentionally context-pack-first too: run `madar generate .`, install with `madar codex install`, and start broad codebase work with `madar pack "" --task explain` before raw file search. The install writes the Madar-owned AGENTS.md section, `.codex/hooks.json`, `.codex/madar-user-prompt-submit.cjs`, and a marker-owned strict-profile `[mcp_servers.madar]` block in `.codex/config.toml`. That block includes `startup_timeout_sec = 180` for cold large-repository and synchronized-filesystem startup. Re-run the install after upgrading to migrate an older Madar-owned block; user-managed MCP declarations remain untouched. Its `UserPromptSubmit` hook provides model-visible guidance only for local code tasks; it is guidance, not enforcement. Enable it only in a trusted repository, restart or start a new Codex session, use `/hooks` to review and trust the project hook, then verify the server through `/mcp` or `codex mcp list`. `madar doctor` and `madar status` validate on-disk install state only, not live Codex trust or activation. To remove the profile, run `madar codex uninstall`; it removes only Madar-owned AGENTS, hook, script, and marked TOML content while preserving unrelated content. ## MCP Registry metadata @@ -79,6 +79,8 @@ Cached `context_pack` explain responses still refresh the current freshness rece With `--auto-refresh`, filesystem events invalidate the graph immediately and adaptive authoritative reconciliations verify the full watched corpus. Graph-backed MCP requests fail closed while reconciliation is pending/failed or watcher coverage/policy is not trustworthy. Generation policy is versioned and fingerprinted in both `graph.json` and `manifest.json`, so automatic refresh reuses direction, SPI, Git-ignore, symlink, document/non-code, exclusion, extractor, and indexing-threshold settings. Policy drift forces a full rebuild. `madar doctor` and `madar status` expose the local `watcher-state.json` health record. Full behavior and legacy migration are documented in [Auto-refresh and generation policy](../auto-refresh.md). +The stdio transport and MCP discovery stay responsive while initial reconciliation runs in a background worker. Until the watcher reaches `idle` with matching published policy, graph-backed calls return a bounded freshness error that tells the caller to wait or run `madar generate . --update`. + ## Common commands ```bash diff --git a/docs/release.md b/docs/release.md index 4123299d..de7433d7 100644 --- a/docs/release.md +++ b/docs/release.md @@ -46,7 +46,7 @@ Recommended follow-up checks: - confirm `madar --version` prints the version you are about to publish - confirm `madar generate .` completes and refreshes `out/graph.json` - confirm install commands write the expected project files and instructions -- for Codex, confirm `.codex/hooks.json`, `.codex/madar-user-prompt-submit.cjs`, and `.codex/config.toml` exist; only in a trusted repository, restart or open a new session, use `/hooks` to review/trust the project hook, then use `/mcp` or `codex mcp list` to verify the local MCP server +- for Codex, confirm `.codex/hooks.json`, `.codex/madar-user-prompt-submit.cjs`, and `.codex/config.toml` exist, and that the managed MCP block contains `startup_timeout_sec = 180`; only in a trusted repository, restart or open a new session, use `/hooks` to review/trust the project hook, then use `/mcp` or `codex mcp list` to verify the local MCP server - uninstall any agent profile you enabled during the smoke test so the workspace returns to a clean state ## 4. Publish and tag diff --git a/docs/tutorials/agent-quickstarts.md b/docs/tutorials/agent-quickstarts.md index fa465ac7..15a37c80 100644 --- a/docs/tutorials/agent-quickstarts.md +++ b/docs/tutorials/agent-quickstarts.md @@ -109,6 +109,7 @@ This installs the Madar-owned AGENTS.md section, a task-applicable `UserPromptSu Common failure modes: - If `madar status` marks Codex as partial, inspect `.codex/hooks.json`, `.codex/madar-user-prompt-submit.cjs`, and `.codex/config.toml`, then rerun the install. +- If Codex reports that the Madar MCP client timed out after 30 seconds, rerun `madar codex install` and confirm its managed block contains `startup_timeout_sec = 180`. Then run `madar doctor` and `madar status`; if reconciliation remains failed, run `madar generate . --update`. - If Codex ignores the guidance, confirm `AGENTS.md` still contains the Madar-owned rules and that the project hook is trusted in `/hooks`. - `madar codex uninstall` removes only the Madar-owned AGENTS section, hook, script, and marked TOML block; unrelated hooks and TOML configuration remain. diff --git a/src/infrastructure/background-auto-refresh.ts b/src/infrastructure/background-auto-refresh.ts new file mode 100644 index 00000000..04d29444 --- /dev/null +++ b/src/infrastructure/background-auto-refresh.ts @@ -0,0 +1,240 @@ +import { existsSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { Worker } from 'node:worker_threads' + +import { resolveMadarOutputDirectory } from '../shared/workspace.js' +import { + createWatcherState, + readWatcherState, + watcherStatePath, + writeWatcherState, +} from './watcher-state.js' +import { + startGraphAutoRefresh, + type GraphAutoRefreshController, + type WatchLogger, +} from './watch.js' + +// Keep the bootstrap static and pass every path through workerData. This avoids +// shell interpolation and lets Windows workspaces contain spaces safely. +const AUTO_REFRESH_WORKER_SOURCE = String.raw` +const { parentPort, workerData } = require('node:worker_threads') + +if (!parentPort) { + throw new Error('Madar auto-refresh worker has no parent port') +} + +let controller = null +let stopRequested = false + +parentPort.on('message', (message) => { + if (!message || message.type !== 'stop') { + return + } + stopRequested = true + controller?.stop() +}) + +const logger = { + log() {}, + error(message) { + parentPort.postMessage({ type: 'watch-error', message: String(message ?? 'Auto-refresh failed') }) + }, +} + +void (async () => { + const watchModule = await import(workerData.watchModuleUrl) + controller = watchModule.startGraphAutoRefresh( + workerData.watchPath, + workerData.debounceSeconds, + { + noHtml: workerData.noHtml, + logger, + }, + ) + parentPort.postMessage({ type: 'started', initialRebuilt: controller.initialRebuilt }) + if (stopRequested) { + controller.stop() + } + await controller.completed + parentPort.postMessage({ type: 'completed' }) + parentPort.close() +})().catch((error) => { + parentPort.postMessage({ + type: 'worker-error', + message: error instanceof Error ? error.message : String(error), + }) + parentPort.close() +}) +` + +export interface BackgroundAutoRefreshOptions { + noHtml?: boolean + logger?: WatchLogger +} + +export interface BackgroundAutoRefreshDependencies { + /** Internal test seam; production resolves the compiled sibling watch.js. */ + watchModuleUrl?: URL +} + +interface WorkerMessage { + type?: unknown + message?: unknown + initialRebuilt?: unknown +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function markStarting(outputDir: string): void { + writeWatcherState(outputDir, createWatcherState('polling', 0)) +} + +function markFailed(outputDir: string, message: string): void { + const current = readWatcherState(watcherStatePath(outputDir)) + if (current && current.pid !== process.pid) { + return + } + const state = current ?? createWatcherState('polling', 0) + writeWatcherState(outputDir, { + ...state, + status: 'failed', + coverage: 'failed', + failure_reason: message, + next_reconciliation_at: null, + }) +} + +function completedFailureController(message: string): GraphAutoRefreshController { + return { + initialRebuilt: false, + startupComplete: () => true, + failureReason: () => message, + stop() {}, + completed: Promise.resolve(), + } +} + +/** + * Runs the synchronous graph reconciliation/watch loop away from the MCP + * transport thread. The stdio server can therefore finish initialization and + * answer control requests while watcher-state.json keeps graph reads fail-closed. + */ +export function startGraphAutoRefreshInBackground( + watchPath: string, + debounceSeconds = 1, + options: BackgroundAutoRefreshOptions = {}, + dependencies: BackgroundAutoRefreshDependencies = {}, +): GraphAutoRefreshController { + const outputDir = resolveMadarOutputDirectory(watchPath) + try { + markStarting(outputDir) + } catch (error) { + const message = `Unable to mark auto-refresh as starting: ${errorMessage(error)}` + options.logger?.error(message) + return completedFailureController(message) + } + + const watchModuleUrl = dependencies.watchModuleUrl ?? new URL('./watch.js', import.meta.url) + let watchModuleExists = false + try { + watchModuleExists = existsSync(fileURLToPath(watchModuleUrl)) + } catch { + watchModuleExists = false + } + + // Vitest and source-level TypeScript runners do not have a sibling watch.js. + // Keep their existing in-process behavior; published builds always do. + if (!watchModuleExists) { + return startGraphAutoRefresh(watchPath, debounceSeconds, options) + } + + let worker: Worker + try { + worker = new Worker(AUTO_REFRESH_WORKER_SOURCE, { + eval: true, + workerData: { + watchModuleUrl: watchModuleUrl.href, + watchPath, + debounceSeconds, + noHtml: options.noHtml ?? false, + }, + }) + } catch (error) { + const message = `Unable to start auto-refresh worker: ${errorMessage(error)}` + markFailed(outputDir, message) + options.logger?.error(message) + return completedFailureController(message) + } + + let startupComplete = false + let initialRebuilt = false + let failureReason: string | null = null + let stopRequested = false + let settled = false + let resolveCompleted!: () => void + const completed = new Promise((resolvePromise) => { + resolveCompleted = resolvePromise + }) + + function settle(): void { + if (settled) { + return + } + settled = true + resolveCompleted() + } + + function fail(message: string): void { + failureReason = message + startupComplete = true + markFailed(outputDir, message) + options.logger?.error(message) + } + + worker.on('message', (rawMessage: unknown) => { + const message = rawMessage as WorkerMessage + if (message.type === 'started') { + startupComplete = true + initialRebuilt = message.initialRebuilt === true + return + } + if (message.type === 'watch-error' || message.type === 'worker-error') { + fail(typeof message.message === 'string' ? message.message : 'Madar auto-refresh worker failed') + return + } + if (message.type === 'completed') { + settle() + } + }) + worker.once('error', (error) => { + if (!stopRequested) { + fail(`Madar auto-refresh worker crashed: ${errorMessage(error)}`) + } + settle() + }) + worker.once('exit', (code) => { + if (!stopRequested && code !== 0) { + fail(`Madar auto-refresh worker exited with code ${code}`) + } + settle() + }) + + return { + get initialRebuilt() { + return initialRebuilt + }, + startupComplete: () => startupComplete, + failureReason: () => failureReason, + stop() { + if (stopRequested || settled) { + return + } + stopRequested = true + worker.postMessage({ type: 'stop' }) + }, + completed, + } +} diff --git a/src/infrastructure/install.ts b/src/infrastructure/install.ts index 408be0ef..71e3cc1d 100644 --- a/src/infrastructure/install.ts +++ b/src/infrastructure/install.ts @@ -41,6 +41,7 @@ const CODEX_PROMPT_HOOK_SCRIPT_MARKER = '// madar managed Codex UserPromptSubmit // a nested Codex session works without interpolating a shell-sensitive project path. const CODEX_PROMPT_HOOK_COMMAND = `node -e "const fs=require('fs');const path=require('path');let dir=process.cwd();for(;;){const script=path.join(dir,'.codex','madar-user-prompt-submit.cjs');if(fs.existsSync(script)){require(script);break}const parent=path.dirname(dir);if(parent===dir){process.exit(0)}dir=parent}"` export const CODEX_MCP_CONFIG_RELATIVE_PATH = '.codex/config.toml' +export const CODEX_MCP_STARTUP_TIMEOUT_SECONDS = 180 const CODEX_MCP_START_MARKER = '# >>> madar managed mcp >>>' const CODEX_MCP_END_MARKER = '# <<< madar managed mcp <<<' const CODEX_MCP_OWNS_PRECEDING_LINE_ENDING_MARKER = '# madar managed mcp: preceding line ending owned' @@ -2134,6 +2135,7 @@ function renderCodexMcpBlock(lineEnding: string, ownsPrecedingLineEnding = false 'args = ["serve", "--stdio", "--auto-refresh"]', 'env = { MADAR_TOOL_PROFILE = "strict" }', 'enabled = true', + `startup_timeout_sec = ${CODEX_MCP_STARTUP_TIMEOUT_SECONDS}`, CODEX_MCP_END_MARKER, '', ].join(lineEnding) diff --git a/src/infrastructure/watch.ts b/src/infrastructure/watch.ts index d4a68702..68c9154a 100644 --- a/src/infrastructure/watch.ts +++ b/src/infrastructure/watch.ts @@ -112,6 +112,10 @@ export interface WatchOptions extends RebuildCodeOptions { export interface GraphAutoRefreshController { /** Whether the initial incremental reconciliation produced a graph. */ initialRebuilt: boolean + /** Background controllers remain false until their initial reconciliation has settled. */ + startupComplete?(): boolean + /** Returns a background startup/runtime failure that could not be read from watcher-state.json. */ + failureReason?(): string | null /** Stops the watcher and releases its filesystem resources. */ stop(): void /** Resolves once the watcher stops. */ diff --git a/src/runtime/stdio-server.ts b/src/runtime/stdio-server.ts index 2b712176..ed9493e7 100644 --- a/src/runtime/stdio-server.ts +++ b/src/runtime/stdio-server.ts @@ -1,11 +1,12 @@ import { createInterface } from 'node:readline' -import { existsSync, realpathSync, statSync } from 'node:fs' +import { realpathSync, statSync } from 'node:fs' import { basename, dirname, join, resolve } from 'node:path' import type { Readable, Writable } from 'node:stream' import type { ContextSessionState } from '../contracts/context-session.js' import { compareRefs } from '../infrastructure/time-travel.js' -import { startGraphAutoRefresh } from '../infrastructure/watch.js' +import { startGraphAutoRefreshInBackground } from '../infrastructure/background-auto-refresh.js' +import type { GraphAutoRefreshController } from '../infrastructure/watch.js' import { readWatcherStateForGraph } from '../infrastructure/watcher-state.js' import { readStoredGenerationPolicy } from '../infrastructure/generation-policy.js' import { watcherStateBlocksGraphReads } from '../contracts/watcher-state.js' @@ -78,6 +79,8 @@ const AUTO_REFRESH_CONTROL_METHODS = new Set([ 'notifications/initialized', 'logging/setLevel', 'ping', + 'prompts/list', + 'resources/list', 'tools/list', ]) @@ -160,6 +163,8 @@ export interface ServeGraphStdioOptions { input?: Readable output?: Writable errorOutput?: Writable + /** Internal/testing seam for the production background auto-refresh launcher. */ + autoRefreshStarter?: typeof startGraphAutoRefreshInBackground logger?: { log(message?: string): void error(message?: string): void @@ -184,6 +189,43 @@ function graphRootPath(graphPath: string): string | null { } } +function autoRefreshGraphReadiness( + controller: GraphAutoRefreshController, + graphPath: string, +): { ready: boolean; detail: string } { + const startupComplete = controller.startupComplete?.() ?? true + const backgroundFailure = controller.failureReason?.() ?? null + const watcherState = readWatcherStateForGraph(graphPath) + const publishedPolicy = readStoredGenerationPolicy( + graphPath, + join(dirname(graphPath), 'manifest.json'), + ) + const watcherMatchesPublishedPolicy = watcherState !== null + && publishedPolicy !== null + && watcherState.stored_policy_fingerprint === publishedPolicy.fingerprint + const ready = startupComplete + && watcherState !== null + && watcherState.status === 'idle' + && !watcherStateBlocksGraphReads(watcherState) + && watcherMatchesPublishedPolicy + + if (watcherState) { + return { + ready, + detail: `status=${watcherState.status}, coverage=${watcherState.coverage}, policy=${watcherState.policy_match === null ? 'unknown' : watcherState.policy_match ? 'match' : 'mismatch'}, published_policy=${watcherMatchesPublishedPolicy ? 'match' : 'mismatch'}${watcherState.failure_reason ? `, failure=${watcherState.failure_reason}` : ''}`, + } + } + + return { + ready, + detail: backgroundFailure + ? `background startup failed: ${backgroundFailure}` + : startupComplete + ? 'watcher state is unavailable' + : 'background reconciliation is starting', + } +} + function ok(id: string | number | null, result: unknown): StdioResponse { return { jsonrpc: '2.0', id, result } } @@ -925,7 +967,7 @@ export async function serveGraphStdio(options: ServeGraphStdioOptions): Promise< const output = options.output ?? process.stdout const errorOutput = options.errorOutput ?? process.stderr const sessionState = createSessionState() - let autoRefresh: ReturnType | null = null + let autoRefresh: GraphAutoRefreshController | null = null if (options.autoRefresh) { const workspaceRoot = options.workspaceRoot ?? graphRootPath(options.graphPath) @@ -941,17 +983,18 @@ export async function serveGraphStdio(options: ServeGraphStdioOptions): Promise< ) } - autoRefresh = startGraphAutoRefresh(workspace.rootPath, options.autoRefreshDebounceSeconds ?? 1, { + const startAutoRefresh = options.autoRefreshStarter ?? startGraphAutoRefreshInBackground + autoRefresh = startAutoRefresh(workspace.rootPath, options.autoRefreshDebounceSeconds ?? 1, { // The MCP server needs graph.json; avoid regenerating the browser view on // every coalesced agent edit. noHtml: true, - logger: { log() {}, error() {} }, + logger: { + log() {}, + error(message) { + errorOutput.write(`[madar serve] ${message ?? 'Auto-refresh failed'}\n`) + }, + }, }) - if (!autoRefresh.initialRebuilt && !existsSync(options.graphPath)) { - autoRefresh.stop() - await autoRefresh.completed - throw new Error(`Unable to build a graph for ${workspace.rootPath}`) - } } errorOutput.write(`[madar serve] stdio ready for ${options.graphPath}\n`) @@ -985,28 +1028,19 @@ export async function serveGraphStdio(options: ServeGraphStdioOptions): Promise< try { const request = payload as StdioRequest const requestMethod = typeof request.method === 'string' ? request.method : null - if (autoRefresh && requestMethod && !AUTO_REFRESH_CONTROL_METHODS.has(requestMethod)) { - const watcherState = readWatcherStateForGraph(options.graphPath) - const publishedPolicy = readStoredGenerationPolicy( - options.graphPath, - join(dirname(options.graphPath), 'manifest.json'), + const refreshReadiness = autoRefresh && requestMethod + ? autoRefreshGraphReadiness(autoRefresh, options.graphPath) + : null + if (refreshReadiness && !refreshReadiness.ready && requestMethod === 'prompts/list') { + response = ok(requestId(request), { prompts: MCP_PROMPTS }) + } else if (refreshReadiness && !refreshReadiness.ready && requestMethod === 'resources/list') { + response = ok(requestId(request), { resources: [] }) + } else if (refreshReadiness && !refreshReadiness.ready && requestMethod !== null && !AUTO_REFRESH_CONTROL_METHODS.has(requestMethod)) { + response = failure( + requestId(request), + JSONRPC_SERVER_ERROR, + `Madar auto-refresh cannot guarantee a fresh graph (${refreshReadiness.detail}). Wait for reconciliation or run \`madar generate . --update\` before retrying.`, ) - const watcherMatchesPublishedPolicy = watcherState !== null - && publishedPolicy !== null - && watcherState.stored_policy_fingerprint === publishedPolicy.fingerprint - if (!watcherState || watcherState.status !== 'idle' || watcherStateBlocksGraphReads(watcherState) || !watcherMatchesPublishedPolicy) { - const detail = watcherState - ? `status=${watcherState.status}, coverage=${watcherState.coverage}, policy=${watcherState.policy_match === null ? 'unknown' : watcherState.policy_match ? 'match' : 'mismatch'}, published_policy=${watcherMatchesPublishedPolicy ? 'match' : 'mismatch'}${watcherState.failure_reason ? `, failure=${watcherState.failure_reason}` : ''}` - : 'watcher state is unavailable' - response = failure( - requestId(request), - JSONRPC_SERVER_ERROR, - `Madar auto-refresh cannot guarantee a fresh graph (${detail}). Wait for reconciliation or run \`madar generate . --update\` before retrying.`, - ) - } else { - emitResourceNotifications(output, options.graphPath, sessionState) - response = await Promise.resolve(handleStdioRequest(options.graphPath, payload, sessionState)) - } } else { emitResourceNotifications(output, options.graphPath, sessionState) response = await Promise.resolve(handleStdioRequest(options.graphPath, payload, sessionState)) diff --git a/src/runtime/stdio/prompts.ts b/src/runtime/stdio/prompts.ts index 28b06ed4..8f61e4b4 100644 --- a/src/runtime/stdio/prompts.ts +++ b/src/runtime/stdio/prompts.ts @@ -1,10 +1,11 @@ -import { readFileSync, statSync } from 'node:fs' +import { existsSync, readFileSync, statSync } from 'node:fs' import { godNodes, suggestQuestions } from '../../pipeline/analyze.js' import { buildCommunityLabels } from '../../pipeline/community-naming.js' import { MCP_PROMPTS, type McpPromptDefinition } from './definitions.js' import { communitiesFromGraph, loadGraph } from '../serve.js' import { validateGraphPath } from '../../shared/security.js' +import { resolveWorkspaceGraphPath } from '../../shared/workspace.js' interface StdioResponse { jsonrpc: '2.0' @@ -174,6 +175,12 @@ function formatContextPackPromptTaskLine(task: string, prompt: string): string { } export function promptDefinitionsForGraph(graphPath: string): McpPromptDefinition[] { + if (!existsSync(resolveWorkspaceGraphPath(graphPath))) { + // MCP clients may discover prompts while auto-refresh is still creating the + // first graph. Static definitions keep startup responsive; prompts/get + // remains freshness-gated until reconciliation succeeds. + return MCP_PROMPTS + } const context = loadPromptContext(graphPath) const exampleLabels = context.topGodNodes.slice(0, 3).map((node) => node.label) const exampleCommunities = context.topCommunities.slice(0, 2).map((community) => `${community.label} (#${community.communityId})`) diff --git a/src/runtime/stdio/resources.ts b/src/runtime/stdio/resources.ts index a8fa401f..b1f38603 100644 --- a/src/runtime/stdio/resources.ts +++ b/src/runtime/stdio/resources.ts @@ -4,6 +4,7 @@ import type { Writable } from 'node:stream' import { freshnessAnnotations, resourceFreshnessMetadata } from '../freshness.js' import { validateGraphPath } from '../../shared/security.js' +import { resolveWorkspaceGraphPath } from '../../shared/workspace.js' interface StdioResponse { jsonrpc: '2.0' @@ -63,7 +64,14 @@ function resourceUri(name: string): string { } export function resourcesForGraph(graphPath: string): McpResourceDefinition[] { - const safeGraphPath = validateGraphPath(graphPath) + // During a first auto-refresh startup the MCP transport is available before + // graph.json. Resource discovery must return an empty list instead of making + // initialize fail through notification bookkeeping. + const effectiveGraphPath = resolveWorkspaceGraphPath(graphPath) + if (!existsSync(effectiveGraphPath)) { + return [] + } + const safeGraphPath = validateGraphPath(effectiveGraphPath) const outputDir = dirname(safeGraphPath) const candidates: McpResourceDefinition[] = [ { diff --git a/tests/unit/background-auto-refresh.test.ts b/tests/unit/background-auto-refresh.test.ts new file mode 100644 index 00000000..4ef6b009 --- /dev/null +++ b/tests/unit/background-auto-refresh.test.ts @@ -0,0 +1,217 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { setTimeout as delay } from 'node:timers/promises' +import { pathToFileURL } from 'node:url' + +import { describe, expect, it } from 'vitest' + +import { startGraphAutoRefreshInBackground } from '../../src/infrastructure/background-auto-refresh.js' +import { readWatcherStateForGraph } from '../../src/infrastructure/watcher-state.js' +import { serveGraphStdio } from '../../src/runtime/stdio-server.js' + +const SLOW_WATCH_MODULE = ` +import { writeFileSync } from 'node:fs' +import { join } from 'node:path' + +export function startGraphAutoRefresh(watchPath) { + const deadline = Date.now() + 1500 + while (Date.now() < deadline) { + // Deliberately block only the worker thread to model a cold large-repo build. + } + writeFileSync(join(watchPath, 'slow-watch-finished'), '1') + let resolveCompleted + const completed = new Promise((resolve) => { + resolveCompleted = resolve + }) + return { + initialRebuilt: true, + stop() { + resolveCompleted() + }, + completed, + } +} +` + +const FAILING_WATCH_MODULE = ` +export function startGraphAutoRefresh() { + throw new Error('synthetic initial reconciliation failure') +} +` + +async function waitFor(condition: () => boolean, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (condition()) { + return + } + await delay(10) + } + throw new Error('Timed out waiting for expected condition') +} + +describe('background auto-refresh', () => { + it('returns immediately while a slow initial reconciliation runs in a worker', async () => { + const root = mkdtempSync(join(tmpdir(), 'madar-background-refresh-')) + const graphPath = join(root, 'out', 'graph.json') + const watchModulePath = join(root, 'slow-watch.mjs') + const completionMarker = join(root, 'slow-watch-finished') + writeFileSync(watchModulePath, SLOW_WATCH_MODULE, 'utf8') + + try { + const refresh = startGraphAutoRefreshInBackground( + root, + 0.02, + { noHtml: true, logger: { log() {}, error() {} } }, + { watchModuleUrl: pathToFileURL(watchModulePath) }, + ) + + expect(existsSync(completionMarker)).toBe(false) + expect(refresh.startupComplete?.()).toBe(false) + expect(readWatcherStateForGraph(graphPath)).toMatchObject({ + status: 'starting', + coverage: 'unknown', + }) + + let mainThreadTimerRan = false + setTimeout(() => { + mainThreadTimerRan = true + }, 20) + await delay(60) + expect(mainThreadTimerRan).toBe(true) + + refresh.stop() + await refresh.completed + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('completes MCP discovery and fails graph reads closed during slow startup', async () => { + const root = mkdtempSync(join(tmpdir(), 'madar-background-mcp-')) + const graphPath = join(root, 'out', 'graph.json') + const watchModulePath = join(root, 'slow-watch.mjs') + const completionMarker = join(root, 'slow-watch-finished') + const input = new PassThrough() + const output = new PassThrough() + const errorOutput = new PassThrough() + let outputText = '' + output.on('data', (chunk) => { + outputText += chunk.toString('utf8') + }) + writeFileSync(watchModulePath, SLOW_WATCH_MODULE, 'utf8') + + input.end([ + JSON.stringify({ id: 1, method: 'initialize' }), + JSON.stringify({ id: 2, method: 'prompts/list' }), + JSON.stringify({ id: 3, method: 'resources/list' }), + JSON.stringify({ id: 4, method: 'tools/list' }), + JSON.stringify({ id: 5, method: 'stats' }), + ].join('\n')) + + const serverPromise = serveGraphStdio({ + graphPath, + autoRefresh: true, + workspaceRoot: root, + input, + output, + errorOutput, + autoRefreshStarter: (watchPath, debounceSeconds, options) => startGraphAutoRefreshInBackground( + watchPath, + debounceSeconds, + options, + { watchModuleUrl: pathToFileURL(watchModulePath) }, + ), + }) + + try { + await waitFor(() => outputText.includes('"id":5')) + expect(existsSync(completionMarker)).toBe(false) + + const responses = outputText + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { + id?: number + result?: Record + error?: { message?: string } + }) + + expect(responses.find((response) => response.id === 1)?.result).toMatchObject({ + serverInfo: { name: 'madar' }, + }) + expect(responses.find((response) => response.id === 2)?.result).toMatchObject({ prompts: expect.any(Array) }) + expect(responses.find((response) => response.id === 3)?.result).toEqual({ resources: [] }) + expect(responses.find((response) => response.id === 4)?.result).toMatchObject({ tools: expect.any(Array) }) + expect(responses.find((response) => response.id === 5)?.error?.message).toContain( + 'auto-refresh cannot guarantee a fresh graph', + ) + + await serverPromise + expect(readFileSync(watchModulePath, 'utf8')).toContain('Deliberately block only the worker thread') + } finally { + input.destroy() + await serverPromise.catch(() => {}) + rmSync(root, { recursive: true, force: true }) + } + }) + + it('keeps MCP connected and exposes a background startup failure', async () => { + const root = mkdtempSync(join(tmpdir(), 'madar-background-failure-')) + const graphPath = join(root, 'out', 'graph.json') + const watchModulePath = join(root, 'failing-watch.mjs') + const input = new PassThrough() + const output = new PassThrough() + const errorOutput = new PassThrough() + let outputText = '' + let errorText = '' + output.on('data', (chunk) => { + outputText += chunk.toString('utf8') + }) + errorOutput.on('data', (chunk) => { + errorText += chunk.toString('utf8') + }) + writeFileSync(watchModulePath, FAILING_WATCH_MODULE, 'utf8') + + const serverPromise = serveGraphStdio({ + graphPath, + autoRefresh: true, + workspaceRoot: root, + input, + output, + errorOutput, + autoRefreshStarter: (watchPath, debounceSeconds, options) => startGraphAutoRefreshInBackground( + watchPath, + debounceSeconds, + options, + { watchModuleUrl: pathToFileURL(watchModulePath) }, + ), + }) + + try { + input.write(`${JSON.stringify({ id: 11, method: 'initialize' })}\n`) + await waitFor(() => outputText.includes('"id":11')) + await waitFor(() => readWatcherStateForGraph(graphPath)?.status === 'failed') + input.end(`${JSON.stringify({ id: 12, method: 'stats' })}\n`) + await serverPromise + + const responses = outputText + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { id?: number; result?: unknown; error?: { message?: string } }) + expect(responses.find((response) => response.id === 11)?.result).toBeDefined() + expect(responses.find((response) => response.id === 12)?.error?.message).toContain( + 'synthetic initial reconciliation failure', + ) + expect(errorText).toContain('synthetic initial reconciliation failure') + } finally { + input.destroy() + await serverPromise.catch(() => {}) + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/unit/install.test.ts b/tests/unit/install.test.ts index c112e42b..cf55cdda 100644 --- a/tests/unit/install.test.ts +++ b/tests/unit/install.test.ts @@ -21,6 +21,7 @@ import { isInstallPlatform, uninstallCopilotMcp, uninstallSkill, + CODEX_MCP_STARTUP_TIMEOUT_SECONDS, } from '../../src/infrastructure/install.js' import { MCP_TOOLS, activeMcpTools, type McpToolProfile } from '../../src/runtime/stdio/definitions.js' import { normalizeAssertionPath, normalizeAssertionPaths } from './helpers/platform.js' @@ -1669,7 +1670,7 @@ describe('install helpers', () => { withTempDir((projectDir) => { const configPath = join(projectDir, '.codex', 'config.toml') const unrelatedToml = '# Preserve this user comment\r\n[features]\r\nparallel = true\r\n' - const managedBlock = `${CODEX_MCP_START_MARKER}\r\n[mcp_servers.madar]\r\ncommand = "madar"\r\nargs = ["serve", "--stdio", "--auto-refresh"]\r\nenv = { MADAR_TOOL_PROFILE = "strict" }\r\nenabled = true\r\n${CODEX_MCP_END_MARKER}\r\n` + const managedBlock = `${CODEX_MCP_START_MARKER}\r\n[mcp_servers.madar]\r\ncommand = "madar"\r\nargs = ["serve", "--stdio", "--auto-refresh"]\r\nenv = { MADAR_TOOL_PROFILE = "strict" }\r\nenabled = true\r\nstartup_timeout_sec = ${CODEX_MCP_STARTUP_TIMEOUT_SECONDS}\r\n${CODEX_MCP_END_MARKER}\r\n` mkdirSync(join(projectDir, '.codex'), { recursive: true }) writeFileSync(configPath, unrelatedToml, 'utf8') @@ -1705,11 +1706,27 @@ describe('install helpers', () => { expect(installMessage).toContain('.codex/config.toml -> MCP server updated') expect(migrated).toContain('MADAR_TOOL_PROFILE = "strict"') expect(migrated).not.toContain('MADAR_TOOL_PROFILE = "core"') + expect(migrated).toContain(`startup_timeout_sec = ${CODEX_MCP_STARTUP_TIMEOUT_SECONDS}`) expect(countOccurrences(migrated, CODEX_MCP_START_MARKER)).toBe(1) expect(agentsInstall(projectDir, 'codex')).toContain('.codex/config.toml -> MCP server already registered (no change)') }) }) + it('migrates the v0.31.1 owned Codex MCP block to the explicit startup timeout', () => { + withTempDir((projectDir) => { + const configPath = join(projectDir, '.codex', 'config.toml') + const previousBlock = `${CODEX_MCP_START_MARKER}\n[mcp_servers.madar]\ncommand = "madar"\nargs = ["serve", "--stdio", "--auto-refresh"]\nenv = { MADAR_TOOL_PROFILE = "strict" }\nenabled = true\n${CODEX_MCP_END_MARKER}\n` + mkdirSync(join(projectDir, '.codex'), { recursive: true }) + writeFileSync(configPath, previousBlock, 'utf8') + + expect(agentsInstall(projectDir, 'codex')).toContain('.codex/config.toml -> MCP server updated') + expect(readFileSync(configPath, 'utf8')).toContain( + `startup_timeout_sec = ${CODEX_MCP_STARTUP_TIMEOUT_SECONDS}`, + ) + expect(agentsInstall(projectDir, 'codex')).toContain('.codex/config.toml -> MCP server already registered (no change)') + }) + }) + it('restores Codex TOML files that originally had no final line ending', () => { const originalContents = [ 'parallel = true', @@ -1791,6 +1808,7 @@ ${CODEX_MCP_END_MARKER} expect(installed).toContain('args = ["serve", "--stdio", "--auto-refresh"]') expect(installed).toContain('env = { MADAR_TOOL_PROFILE = "strict" }') expect(installed).toContain('enabled = true') + expect(installed).toContain(`startup_timeout_sec = ${CODEX_MCP_STARTUP_TIMEOUT_SECONDS}`) expect(installed).not.toContain('old-madar') }) }) From 285fbc828dd30ea4f236b03a712e1d39886f0e63 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Thu, 16 Jul 2026 09:36:54 +0400 Subject: [PATCH 2/3] chore(release): prepare 0.31.2 --- CHANGELOG.md | 11 +++++++++++ README.md | 10 ++++++---- docs/mcp-registry/server.json | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- sbom.cdx.json | 12 ++++++------ 6 files changed, 28 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4159ce..71dc14c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to the TypeScript package will be documented in this file. +## [0.31.2] - 2026-07-16 + +### Fixed + +- **Codex no longer times out while Madar performs the initial automatic refresh**: the MCP transport becomes responsive immediately while graph reconciliation runs in a background worker, and graph-backed calls remain fail-closed until the watcher reports a ready graph. Worker startup and reconciliation failures remain visible through watcher state, stderr, and MCP freshness errors. Closes #559. +- **Managed Codex profiles now allow large workspaces enough time to start**: new and updated `.codex/config.toml` entries set `startup_timeout_sec = 180` without overwriting unrelated user configuration. + +### Notes + +- After upgrading, rerun `madar codex install` in each Codex workspace to migrate the managed MCP block to the extended startup timeout. + ## [0.31.1] - 2026-07-15 ### Changed diff --git a/README.md b/README.md index 620e2187..77e83037 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Installer details are in the [CLI and MCP reference](https://github.com/mohanagy After upgrading Madar, rerun your agent's install command so its managed profile receives current runtime settings. Older profiles may lack automatic refresh; older Codex profiles may also lack the extended MCP startup window needed by large or synchronized workspaces. -Codex installs set `startup_timeout_sec = 180`. Madar makes the MCP transport available while the initial graph reconciliation runs in a background worker, but graph-backed calls remain safely unavailable until `madar status` reports the watcher as `idle`. +Starting with `0.31.2`, Codex installs set `startup_timeout_sec = 180`. Madar makes the MCP transport available while the initial graph reconciliation runs in a background worker, but graph-backed calls remain safely unavailable until `madar status` reports the watcher as `idle`. ## What Changes for the Agent @@ -187,13 +187,15 @@ Read the [benchmark suite and all dated receipts](https://github.com/mohanagy/ma ## Current Release -Current version: `0.31.1`. +Current version: `0.31.2`. -`0.31.1` rebuilds the public onboarding path and clarifies what each benchmark experiment proves. Runtime behavior is unchanged from `0.31.0`. +`0.31.2` keeps the Codex MCP connection responsive while its initial automatic graph refresh runs, adds an explicit 180-second Codex startup window, and keeps graph-backed answers unavailable until the refreshed graph is ready. + +`0.31.1` rebuilt the public onboarding path and clarified what each benchmark experiment proves. Runtime behavior was unchanged from `0.31.0`. `0.31.0` made code graphs directed by default, separated evidence strength from answer readiness, added bounded context recovery, made indexing completeness explicit, preserved generation policy during automatic refresh, isolated linked-worktree artifacts, and removed benchmark expectations from production retrieval. -Read the full notes in the [0.31.1 changelog](https://github.com/mohanagy/madar/blob/main/CHANGELOG.md#0311---2026-07-15). +Read the full notes in the [0.31.2 changelog](https://github.com/mohanagy/madar/blob/main/CHANGELOG.md#0312---2026-07-16). ## Documentation diff --git a/docs/mcp-registry/server.json b/docs/mcp-registry/server.json index 2ddac926..fc15b7e9 100644 --- a/docs/mcp-registry/server.json +++ b/docs/mcp-registry/server.json @@ -9,13 +9,13 @@ "source": "github", "url": "https://github.com/mohanagy/madar" }, - "version": "0.31.1", + "version": "0.31.2", "packages": [ { "registryType": "npm", "registryBaseUrl": "https://registry.npmjs.org", "identifier": "@lubab/madar", - "version": "0.31.1", + "version": "0.31.2", "runtimeHint": "npx", "transport": { "type": "stdio" diff --git a/package-lock.json b/package-lock.json index 6d4ea364..f6750f16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lubab/madar", - "version": "0.31.1", + "version": "0.31.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lubab/madar", - "version": "0.31.1", + "version": "0.31.2", "license": "MIT", "dependencies": { "@vscode/tree-sitter-wasm": "^0.3.1", diff --git a/package.json b/package.json index 41b8e836..f833ff76 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lubab/madar", - "version": "0.31.1", + "version": "0.31.2", "description": "Stop AI coding agents from rediscovering large TypeScript/Node repos. Madar compiles task-aware local context packs from what runs for this task.", "license": "MIT", "author": "mohanagy", diff --git a/sbom.cdx.json b/sbom.cdx.json index 36bd816d..18b54a28 100644 --- a/sbom.cdx.json +++ b/sbom.cdx.json @@ -2,10 +2,10 @@ "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.5", - "serialNumber": "urn:uuid:456acc7e-1276-4f2d-8b8c-5ae91a55c9b0", + "serialNumber": "urn:uuid:184d19a4-cdcf-4d54-bdb2-7948c1d2c353", "version": 1, "metadata": { - "timestamp": "2026-07-15T15:18:39.220Z", + "timestamp": "2026-07-16T05:07:00.859Z", "lifecycles": [ { "phase": "build" @@ -19,14 +19,14 @@ } ], "component": { - "bom-ref": "@lubab/madar@0.31.1", + "bom-ref": "@lubab/madar@0.31.2", "type": "library", "name": "@lubab/madar", - "version": "0.31.1", + "version": "0.31.2", "scope": "required", "author": "mohanagy", "description": "Stop AI coding agents from rediscovering large TypeScript/Node repos. Madar compiles task-aware local context packs from what runs for this task.", - "purl": "pkg:npm/%40lubab/madar@0.31.1", + "purl": "pkg:npm/%40lubab/madar@0.31.2", "properties": [], "externalReferences": [ { @@ -3296,7 +3296,7 @@ ], "dependencies": [ { - "ref": "@lubab/madar@0.31.1", + "ref": "@lubab/madar@0.31.2", "dependsOn": [ "@vscode/tree-sitter-wasm@0.3.1", "fflate@0.8.3", From 6b1c981221e4850fa584ccebe97f776ed8b13656 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Thu, 16 Jul 2026 09:53:18 +0400 Subject: [PATCH 3/3] fix: harden background refresh readiness --- README.md | 2 +- docs/auto-refresh.md | 2 +- src/infrastructure/background-auto-refresh.ts | 26 ++-- src/runtime/stdio-server.ts | 3 +- tests/unit/background-auto-refresh.test.ts | 139 ++++++++++++++++-- 5 files changed, 149 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 77e83037..8978107d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Installer details are in the [CLI and MCP reference](https://github.com/mohanagy After upgrading Madar, rerun your agent's install command so its managed profile receives current runtime settings. Older profiles may lack automatic refresh; older Codex profiles may also lack the extended MCP startup window needed by large or synchronized workspaces. -Starting with `0.31.2`, Codex installs set `startup_timeout_sec = 180`. Madar makes the MCP transport available while the initial graph reconciliation runs in a background worker, but graph-backed calls remain safely unavailable until `madar status` reports the watcher as `idle`. +Starting with `0.31.2`, Codex installs set `startup_timeout_sec = 180`. Madar makes the MCP transport available while the initial graph reconciliation runs in a background worker. Graph-backed calls resume only after startup completes, watcher health is non-blocking with complete coverage, and the idle watcher's policy matches the published graph and manifest; `idle` alone is not a readiness guarantee. ## What Changes for the Agent diff --git a/docs/auto-refresh.md b/docs/auto-refresh.md index 6e87003d..5485ab1a 100644 --- a/docs/auto-refresh.md +++ b/docs/auto-refresh.md @@ -36,7 +36,7 @@ The local `watcher-state.json` beside `graph.json` is written atomically and inc - pending/failure details; and - stored/current policy fingerprints and match state. -`madar doctor` and `madar status` render those fields. During an auto-refresh MCP session, graph-backed prompts, resources, completions, and tool calls are refused while state is pending, reconciling, failed, incomplete, or policy-mismatched. Retry after the state returns to `idle`; if it remains failed, run `madar generate . --update` and inspect `madar status`. +`madar doctor` and `madar status` render those fields. During an auto-refresh MCP session, graph-backed prompts, resources, completions, and tool calls are refused while state is starting, pending, reconciling, failed, incomplete, or policy-mismatched. Retry after the state returns to `idle`; if it remains failed, run `madar generate . --update` and inspect `madar status`. MCP initialization, ping, and list/discovery requests remain responsive during `starting` and `reconciling`. This lets an agent connect without waiting for a cold large-repository build while preserving the same freshness boundary for every graph answer. diff --git a/src/infrastructure/background-auto-refresh.ts b/src/infrastructure/background-auto-refresh.ts index 04d29444..3f36881c 100644 --- a/src/infrastructure/background-auto-refresh.ts +++ b/src/infrastructure/background-auto-refresh.ts @@ -93,18 +93,22 @@ function markStarting(outputDir: string): void { } function markFailed(outputDir: string, message: string): void { - const current = readWatcherState(watcherStatePath(outputDir)) - if (current && current.pid !== process.pid) { - return + try { + const current = readWatcherState(watcherStatePath(outputDir)) + if (current && current.pid !== process.pid) { + return + } + const state = current ?? createWatcherState('polling', 0) + writeWatcherState(outputDir, { + ...state, + status: 'failed', + coverage: 'failed', + failure_reason: message, + next_reconciliation_at: null, + }) + } catch { + // Controller state and stderr remain available when persistence is not. } - const state = current ?? createWatcherState('polling', 0) - writeWatcherState(outputDir, { - ...state, - status: 'failed', - coverage: 'failed', - failure_reason: message, - next_reconciliation_at: null, - }) } function completedFailureController(message: string): GraphAutoRefreshController { diff --git a/src/runtime/stdio-server.ts b/src/runtime/stdio-server.ts index ed9493e7..733f4f67 100644 --- a/src/runtime/stdio-server.ts +++ b/src/runtime/stdio-server.ts @@ -204,6 +204,7 @@ function autoRefreshGraphReadiness( && publishedPolicy !== null && watcherState.stored_policy_fingerprint === publishedPolicy.fingerprint const ready = startupComplete + && backgroundFailure === null && watcherState !== null && watcherState.status === 'idle' && !watcherStateBlocksGraphReads(watcherState) @@ -212,7 +213,7 @@ function autoRefreshGraphReadiness( if (watcherState) { return { ready, - detail: `status=${watcherState.status}, coverage=${watcherState.coverage}, policy=${watcherState.policy_match === null ? 'unknown' : watcherState.policy_match ? 'match' : 'mismatch'}, published_policy=${watcherMatchesPublishedPolicy ? 'match' : 'mismatch'}${watcherState.failure_reason ? `, failure=${watcherState.failure_reason}` : ''}`, + detail: `status=${watcherState.status}, coverage=${watcherState.coverage}, policy=${watcherState.policy_match === null ? 'unknown' : watcherState.policy_match ? 'match' : 'mismatch'}, published_policy=${watcherMatchesPublishedPolicy ? 'match' : 'mismatch'}${watcherState.failure_reason ? `, failure=${watcherState.failure_reason}` : ''}${backgroundFailure ? `, background_failure=${backgroundFailure}` : ''}`, } } diff --git a/tests/unit/background-auto-refresh.test.ts b/tests/unit/background-auto-refresh.test.ts index 4ef6b009..6338607e 100644 --- a/tests/unit/background-auto-refresh.test.ts +++ b/tests/unit/background-auto-refresh.test.ts @@ -8,8 +8,11 @@ import { pathToFileURL } from 'node:url' import { describe, expect, it } from 'vitest' import { startGraphAutoRefreshInBackground } from '../../src/infrastructure/background-auto-refresh.js' -import { readWatcherStateForGraph } from '../../src/infrastructure/watcher-state.js' +import { generateGraph } from '../../src/infrastructure/generate.js' +import { readStoredGenerationPolicy } from '../../src/infrastructure/generation-policy.js' +import { createWatcherState, readWatcherStateForGraph, writeWatcherState } from '../../src/infrastructure/watcher-state.js' import { serveGraphStdio } from '../../src/runtime/stdio-server.js' +import type { GraphAutoRefreshController } from '../../src/infrastructure/watch.js' const SLOW_WATCH_MODULE = ` import { writeFileSync } from 'node:fs' @@ -41,6 +44,16 @@ export function startGraphAutoRefresh() { } ` +const DELAYED_FAILING_WATCH_MODULE = ` +export function startGraphAutoRefresh() { + const deadline = Date.now() + 250 + while (Date.now() < deadline) { + // Give the parent time to make watcher-state persistence unavailable. + } + throw new Error('synthetic failure with unavailable watcher state') +} +` + async function waitFor(condition: () => boolean, timeoutMs = 2_000): Promise { const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { @@ -52,6 +65,22 @@ async function waitFor(condition: () => boolean, timeoutMs = 2_000): Promise { it('returns immediately while a slow initial reconciliation runs in a worker', async () => { const root = mkdtempSync(join(tmpdir(), 'madar-background-refresh-')) @@ -98,18 +127,21 @@ describe('background auto-refresh', () => { const output = new PassThrough() const errorOutput = new PassThrough() let outputText = '' + let refreshController: GraphAutoRefreshController | null = null output.on('data', (chunk) => { outputText += chunk.toString('utf8') }) + writeFileSync(join(root, 'main.ts'), 'export const value = 1\n', 'utf8') + generateGraph(root, { noHtml: true }) writeFileSync(watchModulePath, SLOW_WATCH_MODULE, 'utf8') - input.end([ + input.write(`${[ JSON.stringify({ id: 1, method: 'initialize' }), JSON.stringify({ id: 2, method: 'prompts/list' }), JSON.stringify({ id: 3, method: 'resources/list' }), JSON.stringify({ id: 4, method: 'tools/list' }), JSON.stringify({ id: 5, method: 'stats' }), - ].join('\n')) + ].join('\n')}\n`) const serverPromise = serveGraphStdio({ graphPath, @@ -118,12 +150,15 @@ describe('background auto-refresh', () => { input, output, errorOutput, - autoRefreshStarter: (watchPath, debounceSeconds, options) => startGraphAutoRefreshInBackground( - watchPath, - debounceSeconds, - options, - { watchModuleUrl: pathToFileURL(watchModulePath) }, - ), + autoRefreshStarter: (watchPath, debounceSeconds, options) => { + refreshController = startGraphAutoRefreshInBackground( + watchPath, + debounceSeconds, + options, + { watchModuleUrl: pathToFileURL(watchModulePath) }, + ) + return refreshController + }, }) try { @@ -150,7 +185,18 @@ describe('background auto-refresh', () => { 'auto-refresh cannot guarantee a fresh graph', ) + await waitFor(() => refreshController?.startupComplete?.() === true) + expect(existsSync(completionMarker)).toBe(true) + publishReadyWatcherState(root, graphPath) + input.end(`${JSON.stringify({ id: 6, method: 'stats' })}\n`) await serverPromise + const readyResponses = outputText + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { id?: number; result?: string; error?: unknown }) + expect(readyResponses.find((response) => response.id === 6)?.result).toContain('Nodes:') + expect(readyResponses.find((response) => response.id === 6)?.error).toBeUndefined() expect(readFileSync(watchModulePath, 'utf8')).toContain('Deliberately block only the worker thread') } finally { input.destroy() @@ -214,4 +260,79 @@ describe('background auto-refresh', () => { rmSync(root, { recursive: true, force: true }) } }) + + it('does not let a controller failure reuse stale ready watcher state', async () => { + const root = mkdtempSync(join(tmpdir(), 'madar-background-stale-state-')) + const graphPath = join(root, 'out', 'graph.json') + const input = new PassThrough() + const output = new PassThrough() + const errorOutput = new PassThrough() + let outputText = '' + output.on('data', (chunk) => { + outputText += chunk.toString('utf8') + }) + writeFileSync(join(root, 'main.ts'), 'export const value = 1\n', 'utf8') + generateGraph(root, { noHtml: true }) + publishReadyWatcherState(root, graphPath) + input.end([ + JSON.stringify({ id: 21, method: 'initialize' }), + JSON.stringify({ id: 22, method: 'stats' }), + ].join('\n')) + + try { + await serveGraphStdio({ + graphPath, + autoRefresh: true, + workspaceRoot: root, + input, + output, + errorOutput, + autoRefreshStarter: () => ({ + initialRebuilt: false, + startupComplete: () => true, + failureReason: () => 'synthetic startup persistence failure', + stop() {}, + completed: Promise.resolve(), + }), + }) + const responses = outputText + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { id?: number; result?: unknown; error?: { message?: string } }) + expect(responses.find((response) => response.id === 21)?.result).toBeDefined() + expect(responses.find((response) => response.id === 22)?.error?.message).toContain( + 'synthetic startup persistence failure', + ) + } finally { + input.destroy() + rmSync(root, { recursive: true, force: true }) + } + }) + + it('keeps controller and stderr failure reporting when watcher-state persistence disappears', async () => { + const root = mkdtempSync(join(tmpdir(), 'madar-background-persistence-')) + const outputDir = join(root, 'out') + const watchModulePath = join(root, 'delayed-failure.mjs') + const errors: string[] = [] + writeFileSync(watchModulePath, DELAYED_FAILING_WATCH_MODULE, 'utf8') + + try { + const refresh = startGraphAutoRefreshInBackground( + root, + 0.02, + { noHtml: true, logger: { log() {}, error(message) { errors.push(String(message)) } } }, + { watchModuleUrl: pathToFileURL(watchModulePath) }, + ) + rmSync(outputDir, { recursive: true, force: true }) + writeFileSync(outputDir, 'watcher state cannot be persisted here', 'utf8') + + await waitFor(() => typeof refresh.failureReason?.() === 'string', 5_000) + await refresh.completed + expect(refresh.failureReason?.()).toContain('synthetic failure with unavailable watcher state') + expect(errors.join('\n')).toContain('synthetic failure with unavailable watcher state') + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) })