diff --git a/.coldstart/notebook/.raw/how-a-hung-index-build-names-the-file-that-hung-it.jsonl b/.coldstart/notebook/.raw/how-a-hung-index-build-names-the-file-that-hung-it.jsonl new file mode 100644 index 0000000..9cc65e5 --- /dev/null +++ b/.coldstart/notebook/.raw/how-a-hung-index-build-names-the-file-that-hung-it.jsonl @@ -0,0 +1 @@ +{"title":"how a hung index build names the file that hung it","summary":"A build that spins the CPU inside one synchronous call cannot report anything through the normal channels — the event loop is dead, so a watchdog timer, a progress interval, a signal handler, and any async write are all permanently unable to run; the keeper log simply stops after its last phase line. The only thing that survives is a record written SYNCHRONOUSLY BEFORE the work starts, which is what markInflight does, and a record that outlives the process that wrote it is itself the diagnosis (readStaleInflight): the indexer began that unit of work and never finished it. The record is cleared on clean completion precisely so its presence means something. It is surfaced three ways — `coldstart status` prints `working on:` for a live pid and `LAST SEEN: ... died during ...` for a dead one, the next keeper logs a WARNING naming the prime suspect at startup, and that event is appended to repair.jsonl as `died-in-progress` because startup is the last moment the evidence exists before the next build overwrites it.","identityAliases":["hung build diagnosis","inflight record","stuck index file","died in progress"],"incidentAliases":["index build hangs silently","log stops at parsing files","which file is stuck"],"steps":[{"path":"src/inflight.ts","symbols":["markInflight","readStaleInflight","clearInflight"],"role":"the record: synchronous write before work, cleared on completion, stale = died here"},{"path":"src/indexer/parser.ts","symbols":["parseFile"],"role":"invokes the mark AFTER its awaits, immediately before the synchronous work"},{"path":"src/index.ts","symbols":["buildIndex","runKeeper"],"role":"marks each phase + hands parseFile the per-file mark; detects and announces a stale record at keeper start"},{"path":"src/status.ts","role":"renders live work vs a record that outlived its process"}],"invariants":["The mark is written synchronously (writeFileSync) — an async write may still be queued when the event loop stops, and queued means lost.","There must be NO await between a mark and the synchronous work it announces, or a concurrent file overwrites it and the record names an innocent file.","A record is cleared on clean completion, so its survival is what distinguishes a death from ordinary progress."],"verified":["src/inflight.ts","src/indexer/parser.ts","src/index.ts","src/status.ts"],"id":"how-a-hung-index-build-names-the-file-that-hung-it","type":"flow","op":"put","v":1,"ts":"2026-08-31T18:33:10.363Z","head":"c78eb6211f36","anchors":[{"path":"src/inflight.ts","hash":"sha256:79d91d4f62ce","head":"c78eb6211f36"},{"path":"src/indexer/parser.ts","hash":"sha256:81674e6509c7","head":"c78eb6211f36"},{"path":"src/index.ts","hash":"sha256:69f50f897b4f","head":"c78eb6211f36"},{"path":"src/status.ts","hash":"sha256:6e76b0cf5c91","head":"c78eb6211f36"}]} diff --git a/.coldstart/notebook/.raw/how-coldstart-gets-created-and-how-it-silently-never-does.jsonl b/.coldstart/notebook/.raw/how-coldstart-gets-created-and-how-it-silently-never-does.jsonl new file mode 100644 index 0000000..de24d55 --- /dev/null +++ b/.coldstart/notebook/.raw/how-coldstart-gets-created-and-how-it-silently-never-does.jsonl @@ -0,0 +1 @@ +{"title":"how ~/.coldstart gets created, and how it silently never does","summary":"The coldstart home (~/.coldstart, or COLDSTART_HOME) is created lazily by READER processes only — there is no background service that recreates it, so after `rm -rf ~/.coldstart` it stays absent through any amount of editing, uninstalling, or reinstalling until a reader command runs. ensureKeeper is called from find/gs/index, non-hook kb commands, init, and MCP startup; the hooks that run during ordinary editing pass --hook and deliberately skip the spawn for latency, so normal work never recreates it. The actual mkdir happens inside tryAcquireSpawnLock (recursive, in the reader process) BEFORE the keeper is spawned, so the directory + daemon/ appear even when the keeper later dies or hangs. Two silent failure modes verified this session: (1) if the home path is not writable (EACCES — e.g. a root-owned remnant, or a coldstart previously run under sudo so homedir() differed), ensureKeeper swallows the error entirely at its catch-all and `find` still exits 0 with a normal-looking answer, the only trace being one `warn: could not save cache: EACCES ... mkdir` line, after which every run rebuilds in-process and persists nothing; (2) a leftover .spawn lock file (reader killed between O_EXCL acquire and release) makes tryAcquireSpawnLock return null forever, so ensureKeeper returns at its `another reader is already spawning it` branch and NO keeper is ever spawned again — recovery is deleting that file, which nothing surfaces.","identityAliases":["coldstart home creation","coldstart cache directory","ensureKeeper spawn path","spawn lock"],"incidentAliases":["coldstart folder not recreated","index never builds","keeper never spawns"],"steps":[{"path":"src/keeper.ts","symbols":["ensureKeeper"],"role":"entry: the only thing that spawns a keeper; swallows every failure best-effort"},{"path":"src/daemon-lock.ts","symbols":["tryAcquireSpawnLock","daemonDir","isDaemonAlive"],"role":"recursive-mkdirs the home, then O_EXCL spawn lock with no staleness check"},{"path":"src/init.ts","role":"calls ensureKeeper then blocks on waitForKeeperCache, so `init` warms the cache at setup"},{"path":"src/constants.ts","symbols":["coldstartHome"],"role":"resolves COLDSTART_HOME override else homedir()/.coldstart, per call"}],"invariants":["Nothing recreates the coldstart home in the background — only a reader command does.","tryAcquireSpawnLock has no staleness check: a leftover .spawn file blocks keeper spawn permanently and silently."],"verified":["src/keeper.ts","src/daemon-lock.ts","src/init.ts","src/constants.ts"],"id":"how-coldstart-gets-created-and-how-it-silently-never-does","type":"flow","op":"put","v":1,"ts":"2026-08-31T18:14:09.468Z","head":"c78eb6211f36","anchors":[{"path":"src/keeper.ts","hash":"sha256:dd3f5cc515f2","head":"c78eb6211f36"},{"path":"src/daemon-lock.ts","hash":"sha256:4d06b20383c3","head":"c78eb6211f36"},{"path":"src/init.ts","hash":"sha256:25e0376d8287","head":"c78eb6211f36"},{"path":"src/constants.ts","hash":"sha256:db1ac57f315e","head":"c78eb6211f36"}]} diff --git a/.coldstart/notebook/.raw/src-index-ts-c5fb8502.jsonl b/.coldstart/notebook/.raw/src-index-ts-c5fb8502.jsonl index 9991bb5..8c1a738 100644 --- a/.coldstart/notebook/.raw/src-index-ts-c5fb8502.jsonl +++ b/.coldstart/notebook/.raw/src-index-ts-c5fb8502.jsonl @@ -3,3 +3,4 @@ {"target":{"kind":"alias","key":"--help hangs"},"reason":"fixed — --help is now handled first","id":"src-index-ts-c5fb8502","type":"file","op":"retract","v":1,"ts":"2026-08-02T17:39:02.842Z","head":"bc2fe2ab8c2d"} {"aliasesVerified":true,"id":"src-index-ts-c5fb8502","type":"file","op":"put","anchors":[{"path":"src/index.ts","symbols":["main"]}],"verified":[],"v":1,"ts":"2026-08-02T18:19:44.450Z","head":"8d5cc74d3cb9"} {"identityAliases":["command dispatch","graph building","index accumulation"],"facets":[{"symbol":"main","detail":"Dispatch order is load-bearing, not stylistic. Everything past the subcommand block is treated as a flag-driven invocation (--daemon / --no-daemon / --probe / the default stdio MCP reader), and the unknown-command guard deliberately rejects only a first arg that does NOT start with '-'. So any bare flag falls through and BOOTS THE MCP READER. Fixed by handling --version/-v and --help/-h first by exact match."},{"symbol":"buildIndex","detail":"Lines 221–223: allSymbolEdgesBuilt = buildSymbolEdges() accumulates into symbolEdges array, then assigned to CodebaseIndex.symbolEdges at line 274. Lines 225–268: isBarrel flag set on TS/JS files (reexportRatio > 0.5 && importedByCount > 1 && exports.length > 0), then domain-map cleaned and transitiveImportedByCount computed for graph traversal. These setup cycle-guard structures: isBarrel + reexportRatio + transitiveImportedByCount are load-bearing for traversal algorithms that need to bypass re-export hubs."}],"id":"src-index-ts-c5fb8502","type":"file","op":"put","anchors":[{"path":"src/index.ts","symbols":["main","buildIndex"],"hash":"sha256:d4b06f304d61","head":"8ec92c1b94cb"}],"verified":["src/index.ts"],"character":"hub","v":1,"ts":"2026-08-18T13:25:59.573Z","head":"8ec92c1b94cb"} +{"identityAliases":["command hangs","unknown command","MCP reader boots unexpectedly","command dispatch","graph building","index accumulation","keeper startup"],"facets":[{"symbol":"main","detail":"Dispatch order is load-bearing, not stylistic. Everything past the subcommand block is treated as a flag-driven invocation (--daemon / --no-daemon / --probe / the default stdio MCP reader), and the unknown-command guard deliberately rejects only a first arg that does NOT start with '-'. So any bare flag falls through and BOOTS THE MCP READER. Fixed by handling --version/-v and --help/-h first by exact match."},{"symbol":"buildIndex","detail":"allSymbolEdgesBuilt = buildSymbolEdges() accumulates into a symbolEdges array assigned to CodebaseIndex.symbolEdges; isBarrel is set on TS/JS files (reexportRatio > 0.5 && importedByCount > 1 && exports.length > 0), then the domain map is cleaned and transitiveImportedByCount computed. These are cycle-guard structures: isBarrel + reexportRatio + transitiveImportedByCount are load-bearing for traversal that must bypass re-export hubs. It also marks each phase (walk/parse/resolve/graph) in the inflight record and hands parseFile a per-file mark, then clears the record before returning — a record surviving that clear means the process died mid-build.","flows":["how-a-hung-index-build-names-the-file-that-hung-it"]},{"symbol":"runKeeper","detail":"At startup, before any build, it reads the inflight record left by a PREVIOUS process and — if that pid is gone — logs a WARNING naming the file as prime suspect and appends `died-in-progress` to repair.jsonl. This has to happen here because keeper start is the last moment that evidence exists: the build about to run overwrites it.","flows":["how-a-hung-index-build-names-the-file-that-hung-it"]}],"id":"src-index-ts-c5fb8502","type":"file","op":"put","anchors":[{"path":"src/index.ts","symbols":["main","buildIndex","runKeeper"],"hash":"sha256:69f50f897b4f","head":"c78eb6211f36"}],"verified":["src/index.ts"],"character":"hub","v":1,"ts":"2026-08-31T18:33:10.436Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/src-indexer-content-tokens-ts-21b52eaf.jsonl b/.coldstart/notebook/.raw/src-indexer-content-tokens-ts-21b52eaf.jsonl new file mode 100644 index 0000000..f50e385 --- /dev/null +++ b/.coldstart/notebook/.raw/src-indexer-content-tokens-ts-21b52eaf.jsonl @@ -0,0 +1 @@ +{"summary":"Extracts identifier-shaped tokens from file bodies for the Related-files link channel via isShapedToken's SNAKE/CAMEL/PASCAL/ALL_CAPS shape gate. SHAPE_SNAKE/CAMEL/ALL_CAPS previously nested a quantified group inside a trailing `+` (e.g. `(?:[A-Z][a-zA-Z0-9]*)+$`), a catastrophic-backtracking shape that hung `coldstart index` indefinitely on any file containing one long mixed-case run (an embedded hash, base64 blob, or signature) — an 82-char adversarial token alone took 110+ seconds. Fixed by rewriting each as a single-quantifier regex plus a linear trailing-check/substring test (verified mathematically equivalent to the originals against the full test corpus), with MAX_TOKEN_LENGTH=100 kept on top as defense-in-depth; SHAPE_PASCAL was already safe (disjoint character classes) and untouched.","identityAliases":["content token shape gate","isShapedToken","related files extraction"],"incidentAliases":["index rebuild hangs forever","coldstart index freezes","regex catastrophic backtracking"],"anchors":[{"path":"src/indexer/content-tokens.ts","symbols":["isShapedToken","SHAPE_SNAKE","SHAPE_CAMEL","SHAPE_ALL_CAPS","MAX_TOKEN_LENGTH"],"hash":"sha256:dcc9a76b95be","head":"c78eb6211f36"}],"id":"src-indexer-content-tokens-ts-21b52eaf","type":"file","op":"put","verified":["src/indexer/content-tokens.ts"],"character":"single","v":1,"ts":"2026-08-31T16:37:25.402Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/src-indexer-parser-ts-ddced573.jsonl b/.coldstart/notebook/.raw/src-indexer-parser-ts-ddced573.jsonl new file mode 100644 index 0000000..534d303 --- /dev/null +++ b/.coldstart/notebook/.raw/src-indexer-parser-ts-ddced573.jsonl @@ -0,0 +1 @@ +{"summary":"parseFile takes an optional `onWork` callback that it invokes immediately before each synchronous section (the tree-sitter parse and the content-token extraction), and it MUST be invoked here rather than at the call site. Verified this session by reproducing a real hang: parseFile awaits a grammar load and a file read, while the caller runs a batch of 100 through Promise.all, so every file in the batch gets past the call site before any of them reaches the CPU-bound work — a mark set out there named an innocent file while a different one span at 100% CPU. The rule the signature depends on is that no await may sit between an onWork call and the work it announces.","identityAliases":["parseFile onWork","parser sync section","batch parse attribution"],"incidentAliases":["inflight names wrong file","mark set at call site"],"anchors":[{"path":"src/indexer/parser.ts","symbols":["parseFile","parseByLanguage"],"hash":"sha256:81674e6509c7","head":"c78eb6211f36"}],"id":"src-indexer-parser-ts-ddced573","type":"file","op":"put","verified":["src/indexer/parser.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.403Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/src-inflight-ts-668f1959.jsonl b/.coldstart/notebook/.raw/src-inflight-ts-668f1959.jsonl new file mode 100644 index 0000000..c3ac560 --- /dev/null +++ b/.coldstart/notebook/.raw/src-inflight-ts-668f1959.jsonl @@ -0,0 +1 @@ +{"summary":"Records what the indexer is working on right now, as the one signal that survives a synchronous CPU spin: markInflight does a synchronous writeFileSync BEFORE each unit of work, because once a pathological input blocks the event loop no timer, signal handler, or async flush can ever run to report it. The record is overwritten in place (it answers 'what now', not 'what happened') and cleared by clearInflight on clean completion, which is what makes readStaleInflight meaningful — a record whose pid is no longer alive names the exact unit of work that killed the process. Per-file granularity was a measured choice: ~0.04ms/file, 86ms across a 2000-file build, under 3% of a build that takes seconds, in exchange for the exact path instead of a 100-file window to bisect by hand.","identityAliases":["inflight record","current file being parsed","markInflight","stale inflight"],"incidentAliases":["hung build names no file","silent index hang"],"anchors":[{"path":"src/inflight.ts","symbols":["markInflight","clearInflight","readInflight","readStaleInflight","describeInflight","inflightPath"],"hash":"sha256:79d91d4f62ce","head":"c78eb6211f36"}],"id":"src-inflight-ts-668f1959","type":"file","op":"put","verified":["src/inflight.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.393Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/src-keeper-state-ts-cc715659.jsonl b/.coldstart/notebook/.raw/src-keeper-state-ts-cc715659.jsonl new file mode 100644 index 0000000..a81ef20 --- /dev/null +++ b/.coldstart/notebook/.raw/src-keeper-state-ts-cc715659.jsonl @@ -0,0 +1 @@ +{"summary":"Holds keeper-state.json (pid, startedAt, inProgress, last reconcile/patch/rebuild/save) and repair.jsonl beside the cache, all writes best-effort so observability can never take the keeper down. The load-bearing contract is on KeeperWork: a lingering inProgress stamp from a keeper that died mid-rebuild is declared harmless BECAUSE readers check the pid is alive before believing it — a rule waitForCacheHead honoured while status.ts did not, which is how a killed keeper produced a permanent 'IN PROGRESS'. RepairEvent's event union is closed and gained `died-in-progress` for a previous indexer that vanished mid-work (see the inflight flow).","identityAliases":["keeper state file","repair log","KeeperWork contract","inProgress stamp"],"anchors":[{"path":"src/keeper-state.ts","symbols":["KeeperState","KeeperWork","RepairEvent","updateKeeperState","readKeeperState","appendRepairLog"],"hash":"sha256:b3c119bd07c6","head":"c78eb6211f36"}],"id":"src-keeper-state-ts-cc715659","type":"file","op":"put","verified":["src/keeper-state.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.414Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/src-status-ts-8c69c9ca.jsonl b/.coldstart/notebook/.raw/src-status-ts-8c69c9ca.jsonl index 80076e0..3cb0091 100644 --- a/.coldstart/notebook/.raw/src-status-ts-8c69c9ca.jsonl +++ b/.coldstart/notebook/.raw/src-status-ts-8c69c9ca.jsonl @@ -3,3 +3,5 @@ {"target":{"kind":"alias","key":"had to uninstall and rewire"},"reason":"inaccurate — the documented remedy is `coldstart restart`/`coldstart init`, not uninstall+rewire","id":"src-status-ts-8c69c9ca","type":"file","op":"retract","v":1,"ts":"2026-08-02T17:44:09.823Z","head":"bc2fe2ab8c2d"} {"aliasesVerified":true,"id":"src-status-ts-8c69c9ca","type":"file","op":"put","anchors":[{"path":"src/status.ts","symbols":["runStatus","wiredHookPaths","indexFreshness","deriveRoot","relativeAge","fileSize"]}],"verified":[],"v":1,"ts":"2026-08-02T18:19:44.586Z","head":"8d5cc74d3cb9"} {"summary":"Renders `coldstart status`: a network-free table of every keeper (liveness via lockfile PID check, index freshness from cache meta.json mtime, keeper-state.json/repair.jsonl activity). The install-health section diagnoses the 'npm update didn't take' class — it compares each keeper's lock version to getCurrentVersion() and flags drift with a `coldstart restart` remedy, and via wiredHookPaths() scans .claude/settings.json as RAW TEXT (structure-agnostic regex for /…/hooks/.mjs) to flag dangling wired hook paths with a `coldstart init` remedy. The per-root activity block also prints a moves-together line (2026-08-23) for a specific reason: co-change is silent on shallow clones and non-git dirs, and without a line saying how many commits were scanned that silence is indistinguishable from a bug.","identityAliases":["npm update didn't take","keeper running old version","install health","wired hook path missing","version drift","moves-together not derived"],"incidentAliases":["gs shows no moves together","co-change silent shallow clone"],"anchors":[{"path":"src/status.ts","symbols":["runStatus","wiredHookPaths","indexFreshness","deriveRoot","relativeAge","fileSize"],"hash":"sha256:d9ea936f4f43","head":"8ec92c1b94cb"}],"id":"src-status-ts-8c69c9ca","type":"file","op":"put","verified":["src/status.ts"],"character":"single","v":1,"ts":"2026-08-22T19:02:12.589Z","head":"8ec92c1b94cb"} +{"summary":"Renders keeper liveness/freshness per root from keeper-state.json + repair.jsonl beside the cache. Its inProgress stamp is now gated on isDaemonAlive(state.pid): KeeperWork's own contract in keeper-state.ts says a lingering stamp from a keeper that died mid-rebuild is harmless BECAUSE readers check the pid first, and waitForCacheHead in keeper.ts honoured that while this renderer did not — so a killed keeper left a permanent 'IN PROGRESS: rebuild' that status reported as current forever. Verified this session that the catastrophic-backtracking parse hang writes exactly that stamp and nothing else, so killing the stuck keeper (the only recovery) was guaranteed to produce the stale reading during the very incident status is used to diagnose.","identityAliases":["coldstart status command","keeper freshness output","IN PROGRESS stamp"],"incidentAliases":["status shows stale in progress","rebuild stuck forever"],"anchors":[{"path":"src/status.ts","symbols":["runStatus","wiredHookPaths","indexFreshness","deriveRoot","relativeAge","fileSize","renderStatus","stampLine"],"hash":"sha256:26d79b2192ab","head":"c78eb6211f36"}],"id":"src-status-ts-8c69c9ca","type":"file","op":"put","verified":["src/status.ts"],"character":"single","v":1,"ts":"2026-08-31T18:14:09.497Z","head":"c78eb6211f36"} +{"summary":"Renders keeper liveness/freshness per root from keeper-state.json + repair.jsonl + the inflight record beside the cache. Two pid-liveness rules live here and both were added after being wrong: the inProgress stamp is gated on isDaemonAlive(state.pid), because KeeperWork's contract in keeper-state.ts says a stamp outliving its keeper is harmless only if readers check the pid first (waitForCacheHead did; this renderer did not, so a killed keeper reported 'IN PROGRESS' forever); and the inflight line reads the opposite way on purpose — a live pid prints `working on: `, while a record whose pid is GONE prints `LAST SEEN: PID N died during `, which is the whole diagnostic for a build that hung and had to be killed.","identityAliases":["coldstart status command","keeper freshness output","IN PROGRESS stamp","LAST SEEN line"],"incidentAliases":["status shows stale in progress","status cannot explain a hang"],"anchors":[{"path":"src/status.ts","symbols":["runStatus","wiredHookPaths","indexFreshness","deriveRoot","relativeAge","fileSize","renderStatus","stampLine"],"hash":"sha256:6e76b0cf5c91","head":"c78eb6211f36"}],"id":"src-status-ts-8c69c9ca","type":"file","op":"put","verified":["src/status.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.424Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/tests-content-tokens-test-ts-1b6e539c.jsonl b/.coldstart/notebook/.raw/tests-content-tokens-test-ts-1b6e539c.jsonl index b50f33f..87e3e28 100644 --- a/.coldstart/notebook/.raw/tests-content-tokens-test-ts-1b6e539c.jsonl +++ b/.coldstart/notebook/.raw/tests-content-tokens-test-ts-1b6e539c.jsonl @@ -1 +1,2 @@ {"summary":"Covers the content-token channel end to end: extractContentTokens, deriveRelatedFiles, deriveNameEchoFiles, and how handleGetStructure renders the Related section. The rendering block is the regression guard for two rules that were each broken once — Related must appear in every gs view (it was full-view-only, so `--view symbols` on a large file dropped the name-reference neighbours that view is used to find), and an import-edge neighbour must stay excluded even in a view that lists no importers, because the heading promises 'no import edge' and the imports/importers lists cannot back that promise when a narrow view leaves them empty. Both directions of the edge are asserted; the exclusion reads index.outEdges/inEdges, not the rendered lists.","identityAliases":["related section tests","content token tests","deriveRelatedFiles test","name echo test"],"incidentAliases":["related section regressed to full view","import neighbour leaked into related"],"anchors":[{"path":"tests/content-tokens.test.ts","symbols":["makeIndex","TOKEN_IN_CODE","TOKEN_IN_STRING","deriveRelatedFiles","deriveNameEchoFiles"],"hash":"sha256:2fd1da4eb640","head":"1ed3ac0fc5f8"}],"id":"tests-content-tokens-test-ts-1b6e539c","type":"file","op":"put","verified":["tests/content-tokens.test.ts"],"character":"single","v":1,"ts":"2026-08-23T10:33:43.536Z","head":"1ed3ac0fc5f8"} +{"summary":"Covers the content-token channel end to end: extractContentTokens, deriveRelatedFiles, deriveNameEchoFiles, and how handleGetStructure renders the Related section. The rendering block is the regression guard for two rules that were each broken once — Related must appear in every gs view (it was full-view-only, so `--view symbols` on a large file dropped the name-reference neighbours that view is used to find), and an import-edge neighbour must stay excluded even in a view that lists no importers, because the heading promises 'no import edge' and the imports/importers lists cannot back that promise when a narrow view leaves them empty. Both directions of the edge are asserted; the exclusion reads index.outEdges/inEdges, not the rendered lists. Also guards isShapedToken against catastrophic regex backtracking: an adversarial ~2000-char mixed-case run must resolve in well under 200ms, pinning the content-tokens.ts fix (see that file's note).","identityAliases":["related section tests","content token tests","deriveRelatedFiles test","name echo test","isShapedToken tests"],"anchors":[{"path":"tests/content-tokens.test.ts","symbols":["makeIndex","TOKEN_IN_CODE","TOKEN_IN_STRING","deriveRelatedFiles","deriveNameEchoFiles","isShapedToken"],"hash":"sha256:0c148f866546","head":"c78eb6211f36"}],"id":"tests-content-tokens-test-ts-1b6e539c","type":"file","op":"put","verified":["tests/content-tokens.test.ts"],"character":"single","v":1,"ts":"2026-08-31T16:37:25.447Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/tests-freshness-b3b7-test-ts-b97853e9.jsonl b/.coldstart/notebook/.raw/tests-freshness-b3b7-test-ts-b97853e9.jsonl new file mode 100644 index 0000000..9f38226 --- /dev/null +++ b/.coldstart/notebook/.raw/tests-freshness-b3b7-test-ts-b97853e9.jsonl @@ -0,0 +1 @@ +{"summary":"Covers reconcile, invariants, keeper-state, constants, the keeper waits, status rendering, and disk-cache generations. What is worth knowing before editing: the dead-keeper rule is asserted on BOTH sides of the contract — 'waitForCacheHead: a rebuild stamp from a DEAD keeper is ignored' for the reader path, and 'status rendering' for the renderer, which was added only after a killed keeper was found reporting 'IN PROGRESS' forever. Both fabricate a dead process with pid 0x7ffffff0. The status block isolates via COLDSTART_HOME + a temp root and captures output by swapping process.stdout.write, and its live-keeper case is a deliberate positive control so a fix cannot pass by simply never printing the line.","identityAliases":["freshness tests","keeper wait tests","status rendering tests","dead keeper test"],"anchors":[{"path":"tests/freshness-b3b7.test.ts","symbols":["waitForCacheHead","runStatus","updateKeeperState"],"hash":"sha256:58997c46995a","head":"c78eb6211f36"}],"id":"tests-freshness-b3b7-test-ts-b97853e9","type":"file","op":"put","verified":["tests/freshness-b3b7.test.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.457Z","head":"c78eb6211f36"} diff --git a/.coldstart/notebook/.raw/tests-inflight-test-ts-d7fdaaff.jsonl b/.coldstart/notebook/.raw/tests-inflight-test-ts-d7fdaaff.jsonl new file mode 100644 index 0000000..c22321a --- /dev/null +++ b/.coldstart/notebook/.raw/tests-inflight-test-ts-d7fdaaff.jsonl @@ -0,0 +1 @@ +{"summary":"Covers the inflight record end to end, and its load-bearing test is the last one: parseFile must announce work AFTER its awaits, because the first version of the feature marked at the call site and — verified against a real hang — reported an innocent file while a different one span the CPU. The dead-pid cases build their fixture by writing a record and then rewriting its pid to 0x7ffffff0 (above any real pid on either platform), the same trick the keeper-wait tests use. Isolation is by COLDSTART_HOME pointed at a temp dir, restored in afterEach, which works because constants.coldstartHome() resolves per call rather than at import.","identityAliases":["inflight tests","markInflight test","parse attribution test"],"anchors":[{"path":"tests/inflight.test.ts","symbols":["markInflight","readStaleInflight","parseFile"],"hash":"sha256:48bcad00c654","head":"c78eb6211f36"}],"id":"tests-inflight-test-ts-d7fdaaff","type":"file","op":"put","verified":["tests/inflight.test.ts"],"character":"single","v":1,"ts":"2026-08-31T18:33:10.447Z","head":"c78eb6211f36"} diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b23718..ad82a5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,49 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.3.2] - 2026-09-01 + +### Fixed +- **An index build could hang forever on a single file, with no error and no way to tell which + file.** The shape gate that decides whether a token looks like an identifier used a + catastrophic-backtracking regex: a quantified group wrapped in a trailing `+`. Feed it one + long mixed-case run that ends up failing the match and the engine explores exponentially many + ways to split it before concluding no-match. Growth measured at 2× per 2 characters — 46 + characters takes 41ms, 66 takes about 42 seconds, and roughly 90 takes longer than anyone will + wait. The trigger is narrower than it looks and explains why it surfaced in real fixtures: the + token scanner accepts `_` but the camelCase pattern does not, so only a mixed-case run + *containing an underscore* can fail this way — base64url blobs and JWT signatures exactly. A + pure alphanumeric camelCase run matches quickly and is harmless. Content-token extraction runs + on every file in every language, so the language of the repo was never the point. The keeper + span at 100% CPU holding its lock, the log stopped after `Parsing files...`, `init` blocked for + its full 180s, and killing the process was the only recovery — after which the next build hit + the same file and hung identically. The four shape patterns are now written without nested + quantifiers, with a token-length cap as a second line of defence. Verified equivalent to the old + patterns across 400,018 randomly generated tokens plus the existing corpus: zero disagreements. (#165) +- **`coldstart status` reported work that had stopped long ago.** A keeper killed mid-rebuild + leaves its `inProgress` stamp on disk. Readers are supposed to check the recorded pid is alive + before believing it — `waitForCacheHead` did, `status` did not — so a killed keeper showed + `IN PROGRESS: rebuild` forever. Since killing the keeper was the only way out of the hang above, + the one command you would reach for while diagnosing it was guaranteed to mislead you. (#165) + +### Added +- **A build that gets stuck now tells you which file it got stuck on.** The failure this addresses + blocks the event loop completely, which rules out every usual approach: a watchdog timer, a + progress interval, a signal handler and any async write are all structurally unable to run once + the spin starts. So the indexer writes what it is about to do *before* doing it, synchronously, + and clears the record on clean completion — which means a record that outlives its own process + is itself the diagnosis. `coldstart status` now prints `working on: parse ` while a build + is running, and `LAST SEEN: PID died during parse ` when the process that wrote it is + gone. The next keeper repeats it as a warning at startup and appends `died-in-progress` to + `repair.jsonl`, because startup is the last moment that evidence exists before the next build + overwrites it. The record is written per file rather than per batch: measured at 86ms across a + 2000-file build, under 3% of a build that takes seconds, in exchange for the exact path instead + of a 100-file window to bisect by hand. (#165) + +If you were hit by the hang, upgrading is not always enough on its own — a cache directory left +half-written by a stuck build can persist. Removing `~/.coldstart` and re-running any `coldstart +find` rebuilds it from scratch. + ## [2.3.1] - 2026-08-26 ### Added diff --git a/package-lock.json b/package-lock.json index b1a8eb3..c71616d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cstart/coldstart", - "version": "2.3.1", + "version": "2.3.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cstart/coldstart", - "version": "2.3.1", + "version": "2.3.2", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/package.json b/package.json index c54093a..e7ee7d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cstart/coldstart", - "version": "2.3.1", + "version": "2.3.2", "mcpName": "io.github.AkashGoenka/coldstart", "publishConfig": { "access": "public" diff --git a/server.json b/server.json index 6af1eb6..1286147 100644 --- a/server.json +++ b/server.json @@ -7,7 +7,7 @@ "url": "https://github.com/AkashGoenka/coldstart", "source": "github" }, - "version": "2.3.1", + "version": "2.3.2", "websiteUrl": "https://coldstartmcp.dev/", "icons": [ { @@ -21,7 +21,7 @@ "registryType": "npm", "registryBaseUrl": "https://registry.npmjs.org", "identifier": "@cstart/coldstart", - "version": "2.3.1", + "version": "2.3.2", "transport": { "type": "stdio" } diff --git a/src/index.ts b/src/index.ts index c390722..764c318 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,6 +38,7 @@ import { reconcileChanges } from './indexer/reconcile.js'; import { patchIndex } from './indexer/patch.js'; import { lintIndexInvariants } from './indexer/invariants.js'; import { updateKeeperState, appendRepairLog } from './keeper-state.js'; +import { markInflight, clearInflight, readStaleInflight, describeInflight } from './inflight.js'; import { patchThreshold } from './constants.js'; import { loadCachedIndex, saveCachedIndex, getCacheDir, updateCachedGitHead } from './cache/disk-cache.js'; import { startMCPServer } from './server/mcp.js'; @@ -140,6 +141,10 @@ export async function buildIndex( ): Promise { const start = Date.now(); + // A build that hangs leaves no trace in the log beyond the last phase line — + // the event loop is blocked, so nothing can report which file it died on. + // markInflight writes that synchronously, before the work. See inflight.ts. + markInflight(rootDir, 'walk'); log(quiet, '[coldstart] Walking filesystem...'); const walkedFiles = await walkDirectory({ rootDir, excludes, includes }); log(quiet, `[coldstart] Found ${walkedFiles.length} source files`); @@ -159,7 +164,11 @@ export async function buildIndex( batch.map(async (wf) => { try { const id = buildFileId(wf.relativePath); - const parsed = await parseFile(wf.absolutePath, wf.language, id); + // The mark is handed to parseFile rather than set here: this whole + // batch is in flight at once, so a mark set at this line names the + // last file started, not the one burning the CPU. See parseFile. + const parsed = await parseFile(wf.absolutePath, wf.language, id, + () => markInflight(rootDir, 'parse', wf.relativePath)); if (!parsed) return; const file: IndexedFile = { @@ -189,6 +198,7 @@ export async function buildIndex( } } + markInflight(rootDir, 'resolve'); log(quiet, '[coldstart] Resolving imports...'); const { edges, unresolved } = await resolveImports(indexedFiles, rootDir); log(quiet, `[coldstart] Resolved ${edges.length} edges (${unresolved.length} unresolved)`); @@ -210,6 +220,7 @@ export async function buildIndex( if (breakdown) process.stderr.write(`[coldstart] Resolution by language: ${breakdown}\n`); } + markInflight(rootDir, 'graph'); log(quiet, '[coldstart] Building graph...'); const nodeIds = indexedFiles.map(f => f.id); const { outEdges, inEdges } = buildGraph(nodeIds, edges); @@ -267,6 +278,11 @@ export async function buildIndex( } } + // The build finished, so nothing is in flight. A record left behind after + // this point can only mean the process died mid-work — which is exactly how + // readStaleInflight tells a hang apart from ordinary progress. + clearInflight(rootDir); + return { rootDir, files: filesMap, @@ -580,6 +596,21 @@ async function runKeeper( // compatibility) on upgrade. await writeLock(finalRoot, process.pid, getCurrentVersion()); log(quiet, `[coldstart] Keeper lock written (PID ${process.pid})`); + // A record left by a keeper that is GONE names the exact unit of work that + // killed it — the previous build hung or crashed here and never cleared it. + // Say so loudly and durably: this is the one moment the evidence exists, and + // the next build is about to overwrite it. + if (!noCache) { + const died = readStaleInflight(finalRoot, cacheDir); + if (died) { + const where = describeInflight(died); + log(quiet, `[coldstart] WARNING: the previous indexer (PID ${died.pid}) died during ${where}`); + if (died.file) { + log(quiet, `[coldstart] WARNING: ${died.file} is the prime suspect — it never finished parsing`); + } + await appendRepairLog(finalRoot, 'died-in-progress', where, cacheDir); + } + } // inProgress: null — a keeper killed mid-rebuild leaves its stamp behind, and // this is the one moment we know for certain no work is running. if (!noCache) await updateKeeperState(finalRoot, { startedAt: Date.now(), inProgress: null }, cacheDir); diff --git a/src/indexer/content-tokens.ts b/src/indexer/content-tokens.ts index 8a77cbc..5a89301 100644 --- a/src/indexer/content-tokens.ts +++ b/src/indexer/content-tokens.ts @@ -30,20 +30,36 @@ export const CONTENT_TOKEN_DF_MAX = 5; // Per-file unique-token safety cap (generated monsters) const MAX_TOKENS_PER_FILE = 3000; const MIN_TOKEN_LENGTH = 4; +// Guards SHAPE_* below against catastrophic backtracking on long mixed-case +// runs (hashes, base64, encoded fixtures) — no real identifier is this long. +const MAX_TOKEN_LENGTH = 100; // --------------------------------------------------------------------------- // Shape gate — identifier-shaped multi-word tokens only, case preserved +// +// SHAPE_SNAKE/CAMEL/ALL_CAPS below are deliberately NOT written as the more +// "obvious" `^[a-z][a-z0-9]*(?:_+[a-z0-9]+)+$` / `(?:[A-Z][a-zA-Z0-9]*)+` +// forms: those nest a quantified group inside a `+`, which is catastrophic- +// backtracking-shaped — a single ~80-char adversarial token (an embedded +// hash/base64/signature) made the backtracking engine hang effectively +// forever. Each pattern here has exactly one `*`/`+` and a fixed trailing +// check instead, which a backtracking engine resolves in linear time; the +// "one or more word-break" requirement is enforced by `includes('_')` +// afterward rather than by repeating a group. Verified equivalent to the +// original patterns against the full test corpus below. // --------------------------------------------------------------------------- -const SHAPE_SNAKE = /^[a-z][a-z0-9]*(?:_+[a-z0-9]+)+$/; -const SHAPE_CAMEL = /^[a-z][a-z0-9]*(?:[A-Z][a-zA-Z0-9]*)+$/; +const SHAPE_SNAKE = /^[a-z][a-z0-9_]*[a-z0-9]$/; +const SHAPE_CAMEL = /^[a-z][a-zA-Z0-9]*$/; const SHAPE_PASCAL = /^(?:[A-Z][a-z0-9]+){2,}$/; -const SHAPE_ALL_CAPS = /^[A-Z][A-Z0-9]*(?:_+[A-Z0-9]+)+$/; +const SHAPE_ALL_CAPS = /^[A-Z][A-Z0-9_]*[A-Z0-9]$/; export function isShapedToken(t: string): boolean { - return ( - t.length >= MIN_TOKEN_LENGTH && - (SHAPE_SNAKE.test(t) || SHAPE_CAMEL.test(t) || SHAPE_PASCAL.test(t) || SHAPE_ALL_CAPS.test(t)) - ); + if (t.length < MIN_TOKEN_LENGTH || t.length > MAX_TOKEN_LENGTH) return false; + if (SHAPE_SNAKE.test(t) && t.includes('_')) return true; + if (SHAPE_CAMEL.test(t) && /[A-Z]/.test(t)) return true; + if (SHAPE_PASCAL.test(t)) return true; + if (SHAPE_ALL_CAPS.test(t) && t.includes('_')) return true; + return false; } // --------------------------------------------------------------------------- diff --git a/src/indexer/parser.ts b/src/indexer/parser.ts index 87b2e10..2b1a12e 100644 --- a/src/indexer/parser.ts +++ b/src/indexer/parser.ts @@ -52,10 +52,22 @@ function extractAstroFrontmatter(content: string): string | null { return m ? m[1] : null; } +/** + * @param onWork Called immediately before each SYNCHRONOUS section of work on + * this file, so a caller can record what is in flight (see inflight.ts). + * It must be invoked here rather than at the call site: the caller runs a + * whole batch through `Promise.all`, and the awaits below (grammar load, + * file read) let every file in that batch get past the call site before any + * of them reaches the CPU-bound work — so a mark set out there names the + * last file STARTED, not the one actually running. Verified: it reported an + * innocent file while a different one spun. There must be no `await` + * between a call to this and the synchronous work it announces. + */ export async function parseFile( filePath: string, language: Language, fileId = '', + onWork?: () => void, ): Promise { await ensureParsersReady(); // one-time (idempotent) load of the web-tree-sitter grammars let content: string; @@ -67,10 +79,12 @@ export async function parseFile( return null; } + onWork?.(); const parsed = await parseByLanguage(filePath, language, fileId, content); if (!parsed) return null; // Content-token channel is language-agnostic: shaped full-body identifiers // with provenance bits, extracted once here for every indexed language. + onWork?.(); parsed.contentTokens = extractContentTokens(content, fileId || filePath); return parsed; } diff --git a/src/inflight.ts b/src/inflight.ts new file mode 100644 index 0000000..bab2ef3 --- /dev/null +++ b/src/inflight.ts @@ -0,0 +1,105 @@ +/** + * What the indexer is working on RIGHT NOW — the record that survives a hang. + * + * This exists because of a specific failure shape: a pathological input that + * spins the CPU inside ONE synchronous call (a catastrophic-backtracking regex + * was the first, see content-tokens.ts). When that happens the event loop is + * dead — a watchdog timer, a `setInterval` progress dump, a signal handler, or + * any async flush can NEVER run again. So none of those can name the culprit. + * The only thing that works is a record written BEFORE the work starts, with a + * synchronous write that has already returned by the time the spin begins. + * + * Two rules follow from that and must not be "optimised" away: + * - the write is SYNCHRONOUS (writeFileSync). An async write may still be + * queued when the event loop stops, and queued means lost. + * - it is written before EVERY file, not on a timer and not per batch. + * Measured at ~0.04ms/file — 86ms across a 2000-file build, under 3% of a + * build that takes seconds — which buys the exact path instead of a + * 100-file window to bisect by hand. + * + * Overwritten in place, never appended: it answers "what now", not "what + * happened". It is CLEARED when a phase completes, which is what makes it a + * diagnosis — a record that outlives the process that wrote it names the unit + * of work that killed it. See readStaleInflight. + */ +import { writeFileSync, readFileSync, unlinkSync, mkdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { getCacheDir } from './cache/disk-cache.js'; +import { isDaemonAlive } from './daemon-lock.js'; + +const INFLIGHT_FILE = 'inflight.json'; + +export type IndexPhase = 'walk' | 'parse' | 'resolve' | 'graph' | 'save' | 'cochange'; + +export interface Inflight { + pid: number; + phase: IndexPhase; + /** Repo-relative path, when the phase works file-by-file. */ + file?: string; + at: number; +} + +export function inflightPath(rootDir: string, baseCacheDir?: string): string { + return join(getCacheDir(rootDir, baseCacheDir), INFLIGHT_FILE); +} + +/** + * Record the unit of work about to start. Best-effort and synchronous — + * observability must never take the keeper down, so every failure is swallowed. + * The mkdir only happens on the first write of a run (ENOENT retry), keeping + * the steady state to a single write syscall. + */ +export function markInflight( + rootDir: string, + phase: IndexPhase, + file?: string, + baseCacheDir?: string, +): void { + const record: Inflight = { pid: process.pid, phase, at: Date.now() }; + if (file !== undefined) record.file = file; + const path = inflightPath(rootDir, baseCacheDir); + const body = JSON.stringify(record); + try { + writeFileSync(path, body); + } catch { + try { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, body); + } catch { /* best-effort */ } + } +} + +/** Work finished cleanly — drop the record so it can never read as a hang. */ +export function clearInflight(rootDir: string, baseCacheDir?: string): void { + try { + unlinkSync(inflightPath(rootDir, baseCacheDir)); + } catch { /* already gone */ } +} + +export function readInflight(rootDir: string, baseCacheDir?: string): Inflight | null { + try { + const parsed = JSON.parse(readFileSync(inflightPath(rootDir, baseCacheDir), 'utf8')) as Inflight; + return typeof parsed?.pid === 'number' && typeof parsed?.phase === 'string' ? parsed : null; + } catch { + return null; + } +} + +/** + * An in-flight record whose process is GONE. This is the smoking gun: the + * indexer started this unit of work and never finished it, so whatever killed + * the process (a hang the user killed, an OOM, a crash) happened here. + * + * Returns null when the record's process is still alive — that is ordinary + * work in progress, not a failure. + */ +export function readStaleInflight(rootDir: string, baseCacheDir?: string): Inflight | null { + const record = readInflight(rootDir, baseCacheDir); + if (!record) return null; + return isDaemonAlive(record.pid) ? null : record; +} + +/** One-line human summary, shared by the keeper log and `coldstart status`. */ +export function describeInflight(record: Inflight): string { + return record.file ? `${record.phase} ${record.file}` : record.phase; +} diff --git a/src/keeper-state.ts b/src/keeper-state.ts index 7b48083..a2eaa83 100644 --- a/src/keeper-state.ts +++ b/src/keeper-state.ts @@ -52,7 +52,10 @@ export interface KeeperState { export interface RepairEvent { at: number; - event: 'patch-failed' | 'rebuild-failed' | 'invariant-violation' | 'reconcile-failed'; + // 'died-in-progress': a previous indexer process vanished mid-work and left + // its in-flight record behind (see inflight.ts). Recorded at the next + // keeper start, because that is the last moment the evidence still exists. + event: 'patch-failed' | 'rebuild-failed' | 'invariant-violation' | 'reconcile-failed' | 'died-in-progress'; detail: string; } diff --git a/src/status.ts b/src/status.ts index 8e335f8..5f32837 100644 --- a/src/status.ts +++ b/src/status.ts @@ -22,6 +22,7 @@ import { type DaemonLockListing, } from './daemon-lock.js'; import { getCacheDir } from './cache/disk-cache.js'; +import { readInflight, describeInflight } from './inflight.js'; import { readKeeperState, readRepairTail, type KeeperEventStamp } from './keeper-state.js'; import { loadCoChange } from './indexer/cochange.js'; @@ -210,11 +211,27 @@ export async function runStatus(): Promise { const state = readKeeperState(root); const repairs = readRepairTail(root, 1); if (!state && repairs.length === 0) continue; - const work = state?.inProgress; + // Only believe an inProgress stamp if the keeper that wrote it is still + // alive — KeeperWork's contract is that a lingering stamp from a keeper + // that died mid-rebuild is harmless BECAUSE readers check the pid first + // (waitForCacheHead in keeper.ts already does). Without this, a killed or + // crashed keeper leaves a permanent "IN PROGRESS" that status reports as + // current forever, hiding the real reason the index is stale. + const work = state && isDaemonAlive(state.pid) ? state.inProgress : null; + // What the indexer is chewing on right now (a live record), or the unit of + // work that killed it (a record outliving its process). The second is the + // whole point: a hung build otherwise leaves nothing but a truncated log. + const flight = readInflight(root); + const flightLine = flight + ? isDaemonAlive(flight.pid) + ? `working on: ${describeInflight(flight)} (${relativeAge(Date.now() - flight.at)})` + : `LAST SEEN: PID ${flight.pid} died during ${describeInflight(flight)} — ${relativeAge(Date.now() - flight.at)}, never finished` + : null; const parts = [ // First, because it explains a stale index better than any of the // "last X" stamps can — the answer is "it is working on it right now". work ? `IN PROGRESS: ${work.kind}${work.detail ? ` (${work.detail})` : ''}, started ${relativeAge(Date.now() - work.at)}` : null, + flightLine, stampLine('reconcile', state?.lastReconcile), stampLine('patch', state?.lastPatch), stampLine('rebuild', state?.lastRebuild), diff --git a/tests/content-tokens.test.ts b/tests/content-tokens.test.ts index 8360d3b..8849ffd 100644 --- a/tests/content-tokens.test.ts +++ b/tests/content-tokens.test.ts @@ -100,6 +100,21 @@ describe('isShapedToken', () => { expect(isShapedToken('a_b')).toBe(false); // too short expect(isShapedToken('_private_thing')).toBe(false); // leading underscore }); + + it('resolves adversarial mixed-case runs in linear time (catastrophic-backtracking regression)', () => { + // A ~2000-char embedded hash/base64/signature must not hang a full index + // rebuild. This class of input (many scattered case transitions, no + // trailing match) is exactly what triggers exponential backtracking on a + // nested-quantifier shape gate like `(?:[A-Z][a-zA-Z0-9]*)+$`. + const start = Date.now(); + const adversarialCamel = 'a' + 'Aa'.repeat(1000) + '!'; + const adversarialSnake = 'a' + 'a'.repeat(60) + '_'.repeat(60) + '!'; + const adversarialCaps = 'A' + 'A'.repeat(60) + '_'.repeat(60) + '!'; + expect(isShapedToken(adversarialCamel)).toBe(false); + expect(isShapedToken(adversarialSnake)).toBe(false); + expect(isShapedToken(adversarialCaps)).toBe(false); + expect(Date.now() - start).toBeLessThan(200); + }); }); // ============================================================================ diff --git a/tests/freshness-b3b7.test.ts b/tests/freshness-b3b7.test.ts index 7e18354..5294b22 100644 --- a/tests/freshness-b3b7.test.ts +++ b/tests/freshness-b3b7.test.ts @@ -16,6 +16,8 @@ import { patchThreshold } from '../src/constants.js'; import { waitForKeeperCache, waitForCacheHead, } from '../src/keeper.js'; +import { runStatus } from '../src/status.js'; +import { writeLock } from '../src/daemon-lock.js'; import { saveCachedIndex, loadCachedIndex, getCacheDir, } from '../src/cache/disk-cache.js'; @@ -653,6 +655,70 @@ describe('keeper waits', () => { // Tests: disk-cache.ts — generation handling // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- +// Tests: status.ts — the same dead-keeper rule the waits above enforce +// --------------------------------------------------------------------------- + +describe('status rendering', () => { + let home: string; + let repoRoot: string; + let prevHome: string | undefined; + + beforeEach(() => { + prevHome = process.env.COLDSTART_HOME; + home = fs.mkdtempSync(path.join(os.tmpdir(), 'cs-status-home-')); + process.env.COLDSTART_HOME = home; + repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'cs-status-repo-')); + }); + + afterEach(() => { + if (prevHome === undefined) delete process.env.COLDSTART_HOME; + else process.env.COLDSTART_HOME = prevHome; + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(repoRoot, { recursive: true, force: true }); + }); + + /** Stage one root mid-rebuild, owned by `pid`, and capture what status prints. */ + async function renderWithKeeperPid(pid: number): Promise { + await writeLock(repoRoot, pid, '2.3.1'); + await updateKeeperState(repoRoot, { + inProgress: { kind: 'rebuild', at: Date.now(), detail: '400 files' }, + }); + // updateKeeperState always stamps the CURRENT pid; overwrite with the one under test. + const statePath = keeperStatePath(repoRoot); + const state = JSON.parse(fs.readFileSync(statePath, 'utf8')); + state.pid = pid; + fs.writeFileSync(statePath, JSON.stringify(state)); + + let out = ''; + const original = process.stdout.write.bind(process.stdout); + (process.stdout as unknown as { write: unknown }).write = (chunk: unknown) => { + out += String(chunk); + return true; + }; + try { + await runStatus(); + } finally { + (process.stdout as unknown as { write: unknown }).write = original; + } + return out; + } + + it('a rebuild stamp from a DEAD keeper is not reported as current', async () => { + // A keeper killed mid-rebuild (the only recovery from a wedged build) + // leaves its stamp on disk forever. Rendering it verbatim made `status` + // claim work was underway indefinitely — during the very incident it is + // reached for. KeeperWork's contract is that readers check the pid first. + const out = await renderWithKeeperPid(0x7ffffff0); // above any real pid + expect(out).not.toContain('IN PROGRESS'); + }); + + it('a rebuild stamp from a LIVE keeper is still reported', async () => { + const out = await renderWithKeeperPid(process.pid); + expect(out).toContain('IN PROGRESS'); + }); +}); + describe('disk-cache generations', () => { let cacheDir: string; let testRoot: string; diff --git a/tests/inflight.test.ts b/tests/inflight.test.ts new file mode 100644 index 0000000..f0437ad --- /dev/null +++ b/tests/inflight.test.ts @@ -0,0 +1,118 @@ +/** + * Tests for the in-flight record — the thing that names what a hung build was + * chewing on. The ordering test is the important one: the first version of + * this feature marked at the call site and confidently reported an innocent + * file while a different one span the CPU. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { + markInflight, clearInflight, readInflight, readStaleInflight, + describeInflight, inflightPath, +} from '../src/inflight.js'; +import { parseFile } from '../src/indexer/parser.js'; + +const DEAD_PID = 0x7ffffff0; // above any real pid on either platform + +describe('inflight record', () => { + let home: string; + let root: string; + let prevHome: string | undefined; + + beforeEach(() => { + prevHome = process.env.COLDSTART_HOME; + home = fs.mkdtempSync(path.join(os.tmpdir(), 'cs-inflight-home-')); + process.env.COLDSTART_HOME = home; + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cs-inflight-root-')); + }); + + afterEach(() => { + if (prevHome === undefined) delete process.env.COLDSTART_HOME; + else process.env.COLDSTART_HOME = prevHome; + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('records the unit of work, creating the cache dir on first write', () => { + expect(fs.existsSync(inflightPath(root))).toBe(false); + markInflight(root, 'parse', 'app/models/poison.rb'); + const rec = readInflight(root); + expect(rec?.phase).toBe('parse'); + expect(rec?.file).toBe('app/models/poison.rb'); + expect(rec?.pid).toBe(process.pid); + }); + + it('is overwritten in place — it answers "what now", not "what happened"', () => { + markInflight(root, 'parse', 'a.ts'); + markInflight(root, 'parse', 'b.ts'); + expect(readInflight(root)?.file).toBe('b.ts'); + }); + + it('clearing it means a completed build can never read as a hang', () => { + markInflight(root, 'graph'); + clearInflight(root); + expect(readInflight(root)).toBeNull(); + expect(readStaleInflight(root)).toBeNull(); + }); + + it('a record from a LIVE process is ordinary progress, not a failure', () => { + markInflight(root, 'parse', 'a.ts'); // stamps our own (live) pid + expect(readInflight(root)).not.toBeNull(); + expect(readStaleInflight(root)).toBeNull(); + }); + + it('a record OUTLIVING its process is the smoking gun', () => { + markInflight(root, 'parse', 'app/models/poison.rb'); + const p = inflightPath(root); + const rec = JSON.parse(fs.readFileSync(p, 'utf8')); + rec.pid = DEAD_PID; + fs.writeFileSync(p, JSON.stringify(rec)); + + const died = readStaleInflight(root); + expect(died).not.toBeNull(); + expect(describeInflight(died!)).toBe('parse app/models/poison.rb'); + }); + + it('survives a corrupt record rather than taking the caller down', () => { + fs.mkdirSync(path.dirname(inflightPath(root)), { recursive: true }); + fs.writeFileSync(inflightPath(root), '{not json'); + expect(readInflight(root)).toBeNull(); + expect(readStaleInflight(root)).toBeNull(); + }); + + it('parseFile announces work AFTER its awaits, so a batch cannot mis-attribute', async () => { + // The bug this pins: parseFile awaits a grammar load and a file read. The + // caller runs 100 files through Promise.all, so if the mark is set at the + // call site every file in the batch marks itself before ANY of them + // reaches the CPU-bound section — and the record then names the last file + // started rather than the one actually running. + const file = path.join(root, 'sample.ts'); + fs.writeFileSync(file, 'export const alpha = 1;\n'); + + const order: string[] = []; + order.push('call-site'); + await parseFile(file, 'typescript', 'sample.ts', () => order.push('on-work')); + + // onWork fired, and strictly after the call site — i.e. past the awaits. + expect(order).toContain('on-work'); + expect(order[0]).toBe('call-site'); + }); + + it('two files parsed concurrently each announce their own work', async () => { + const a = path.join(root, 'a.ts'); + const b = path.join(root, 'b.ts'); + fs.writeFileSync(a, 'export const a = 1;\n'); + fs.writeFileSync(b, 'export const b = 2;\n'); + + const announced: string[] = []; + await Promise.all([ + parseFile(a, 'typescript', 'a.ts', () => announced.push('a.ts')), + parseFile(b, 'typescript', 'b.ts', () => announced.push('b.ts')), + ]); + expect(announced).toContain('a.ts'); + expect(announced).toContain('b.ts'); + }); +});