merge: integrate verified fixes from #417 through #424

CHANGELOG.md

@@ -5,6 +5,30 @@ Each release of Research-Kit, newest first. The version is `KIT_VERSION` in
 
 ## Unreleased
 
+- `collect-remote` refuses empty repository, topic, client-ref, prior and prefer
+  values by name before creating an output directory or reading a token (#287).
+- Invalid `--snapshot-max-age-ms` values are usage errors with exit 2. Library callers
+  receive `INPUT-SNAPSHOT-MAX-AGE` / `INCOMPLETE` rather than a false stale-snapshot
+  verdict; finite non-negative allowances, including zero, retain their meaning (#343).
+- `decompose --limit` consistently requires a whole number of at least 1 before
+  transport selection. `--max-scrapes 0` remains supported (#320).
+- Auto-collect Git failure diagnostics quote the first `fatal:` or `error:` cause,
+  retaining the final-line fallback for output without either marker (#326).
+- A leading ledger byte-order mark is tolerated while reading, without changing the
+  evidence file or its entry hashes. Tail repair leaves an intact BOM ledger alone
+  and preserves the prefix when dropping a torn append. Handoff gives unreadable lines the tail-repair
+  or restore-from-git remedy; secondary chain failures no longer imply failed travel
+  or lost entries while parsing is unresolved (#296).
+- Finding extraction remembers the winning paragraph start and builds its excerpt
+  forward only until truncation is established. Long unbroken captures avoid the
+  quadratic array insertion while retaining the same finding and excerpt (#334).
+- Recipe loading refuses existing files without dimensions and blank dimension labels,
+  naming the source and line. Decompose reports invalid recipe content with exit 2
+  before writing the map (#356).
+- Oversized panel requests receive the intended 413 JSON refusal before the
+  connection closes. Overflow discards retained body chunks and later data, while
+  a complete JSON body at the existing byte limit remains accepted (#324).
+
 - ZIP archives declare UTF-8 names in both headers, so ordinary extractors preserve non-ASCII paths declared by the package manifest (#345).
 
 - Machine config saves preserve unpatched raw role and posture values, including typos

research-kit/README.md

@@ -568,7 +568,7 @@ node research-kit/bin/selftest.mjs            # all of it
 node research-kit/bin/selftest.mjs gate hook  # just these files
 ```
 
-1845 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
+1861 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
 the assertions settled (ADR-0021), and each test is raced against a watchdog
 (`RESEARCH_KIT_TEST_TIMEOUT`, default 60s).
 

research-kit/bin/collect-remote.mjs

@@ -150,12 +150,9 @@ checkFlagValues(flags, {
   'search-transport': { choices: ['auto', 'serpapi', 'firecrawl-cli'] },
   timeout: { int: true, min: 1 },
   out: {}, workflow: {}, ref: {},
+  repository: {}, topic: {}, 'client-ref': {}, prior: {}, prefer: {},
 }, { exit: EXIT.CANNOT_START });
 
-if (flags['client-ref'] === true) {
-  process.stderr.write('--client-ref needs a value\n');
-  process.exit(EXIT.CANNOT_START);
-}
 const clientReference = checkClientRef(flags['client-ref']);
 if (!clientReference.ok) {
   process.stderr.write(`${redact(clientReference.detail)}\n`);

research-kit/bin/decompose.mjs

@@ -18,7 +18,7 @@ refuseUnknownFlags(flags, ['help', 'topic', 'dry-run', 'force', 'limit', 'max-sc
 // made 1, so `--max-scrapes` with no value quietly paid for one page.
 checkFlagValues(flags, {
   topic: 'value', recipe: 'value', transport: 'value', 'search-transport': 'value',
-  limit: { int: true, min: 0 }, 'max-scrapes': { int: true, min: 0 },
+  limit: { int: true, min: 1 }, 'max-scrapes': { int: true, min: 0 },
 });
 const root = process.cwd();
 
@@ -131,7 +131,7 @@ try {
     process.stderr.write(`${why}\n`);
     process.exit(2);
   }
-  if (err.code === 'UNKNOWN_RECIPE' || err.code === 'INVALID_BUDGET' || err.code === 'SEARCH_PROVIDER_NOT_READY') {
+  if (err.code === 'UNKNOWN_RECIPE' || err.code === 'INVALID_RECIPE' || err.code === 'INVALID_BUDGET' || err.code === 'SEARCH_PROVIDER_NOT_READY') {
     process.stderr.write(`${err.message}\n`);
     process.exit(2);
   }

research-kit/bin/researcher-release.mjs

@@ -54,6 +54,12 @@ function main(argv = process.argv.slice(2)) {
   if (parsed.error) { console.error(`${parsed.error}\n${usage()}`); return 2; }
   let result;
   if (parsed.command === 'validate') {
+    const snapshotMaxAgeMs = parsed.snapshotmaxagems === undefined ? undefined : Number(parsed.snapshotmaxagems);
+    if (parsed.snapshotmaxagems !== undefined
+      && (!parsed.snapshotmaxagems.trim() || !Number.isFinite(snapshotMaxAgeMs) || snapshotMaxAgeMs < 0)) {
+      console.error(`--snapshot-max-age-ms must be a finite non-negative number\n${usage()}`);
+      return 2;
+    }
     if (!parsed.root || !parsed.package || !parsed.registry || !parsed.roles || !parsed.pointers) {
       console.error(usage());
       return 2;
@@ -78,7 +84,7 @@ function main(argv = process.argv.slice(2)) {
       headPath: parsed.head ? path.resolve(parsed.head) : null,
       genesisHash: parsed.genesishash ?? '0'.repeat(64),
       publicKeys,
-      snapshotMaxAgeMs: parsed.snapshotmaxagems === undefined ? undefined : Number(parsed.snapshotmaxagems),
+      snapshotMaxAgeMs,
     });
   } else if (parsed.command === 'conform') {
     if (!parsed.schema) { console.error(usage()); return 2; }

research-kit/lib/auto-collect.mjs

@@ -114,6 +114,11 @@ const tail = (s, n = RUN_OUTPUT_BYTES) => {
   return bytes.length > n ? `[... earlier output truncated ...]\n${bytes.subarray(-n).toString('utf8')}` : text;
 };
 const lastLine = (s) => tail(String(s ?? '').trim().split(/\r?\n/).filter(Boolean).pop() ?? '', 1000);
+// Git puts the cause before multi-line advice; kit run summaries still use lastLine.
+const gitReason = (output) => {
+  const cause = String(output ?? '').split(/\r?\n/).find((line) => /^(fatal|error):/.test(line));
+  return cause === undefined ? lastLine(output) : tail(cause, 1000);
+};
 const posix = (p) => p.split(path.sep).join('/');
 
 /** Legacy diagnostic parser. Collection accounting never trusts human-readable logs. */
@@ -182,7 +187,7 @@ export function createAutoCollect({
       const ahead = await git(['rev-list', '--count', '@{u}..HEAD'], top);
       if (ahead.truncated) return 'could not inspect local commits: Git output was truncated; deliver manually';
       const count = Number(ahead.output.trim());
-      if (ahead.code !== 0) return `could not inspect local commits: ${lastLine(ahead.output)}`;
+      if (ahead.code !== 0) return `could not inspect local commits: ${gitReason(ahead.output)}`;
       if (!/^\d+$/.test(ahead.output.trim()) || !Number.isSafeInteger(count)) {
         return 'could not inspect local commits: invalid Git ahead count; deliver manually';
       }
@@ -195,47 +200,47 @@ export function createAutoCollect({
         }
         const changed = await git(['diff-tree', '--no-commit-id', '--name-only', '-r', '-z', 'HEAD'], top);
         if (changed.truncated) return 'could not inspect pending commit paths: Git output was truncated; deliver manually';
-        if (changed.code !== 0) return `could not inspect pending delivery commit: ${lastLine(changed.output)}`;
+        if (changed.code !== 0) return `could not inspect pending delivery commit: ${gitReason(changed.output)}`;
         const allowed = changed.output.split('\0').filter(Boolean).every((file) => paths.some((base) => (
           file === base || file.startsWith(`${base.replace(/\/$/, '')}/`)
         )));
         if (!allowed) return 'the pending auto-collect commit contains paths outside this request; push it manually';
         const pushed = await git(['push'], top);
-        return pushed.code === 0 ? 'pushed' : `push failed: ${lastLine(pushed.output)}`;
+        return pushed.code === 0 ? 'pushed' : `push failed: ${gitReason(pushed.output)}`;
       }
     }
     const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'research-kit-autocollect-index-'));
     const indexEnv = { ...env, GIT_INDEX_FILE: path.join(scratch, 'index') };
     const isolatedGit = (args) => exec('git', args, { cwd: top, env: indexEnv });
     try {
       const initialized = await isolatedGit(['read-tree', 'HEAD']);
-      if (initialized.code !== 0) return `could not initialize isolated git index: ${lastLine(initialized.output)}`;
+      if (initialized.code !== 0) return `could not initialize isolated git index: ${gitReason(initialized.output)}`;
       for (const p of paths) {
         const r = await isolatedGit(['add', '--', p]);
-        if (r.code !== 0) return `git add ${p} failed: ${lastLine(r.output)}`;
+        if (r.code !== 0) return `git add ${p} failed: ${gitReason(r.output)}`;
       }
       for (const p of paths) {
         if (!(p === 'research' || p.endsWith('/research'))) continue;
         const ledger = path.posix.join(p.slice(0, -'research'.length) || '.', LEDGER);
         if (fs.existsSync(path.join(top, ledger))) {
           const r = await isolatedGit(['add', '-f', '--', ledger]);
-          if (r.code !== 0) return `git add -f ${ledger} failed: ${lastLine(r.output)}`;
+          if (r.code !== 0) return `git add -f ${ledger} failed: ${gitReason(r.output)}`;
         }
       }
       const staged = await isolatedGit(['diff', '--cached', '--quiet']);
-      if (staged.code !== 0 && staged.code !== 1) return `could not inspect isolated git index: ${lastLine(staged.output)}`;
+      if (staged.code !== 0 && staged.code !== 1) return `could not inspect isolated git index: ${gitReason(staged.output)}`;
       const hasChanges = staged.code !== 0;
       if (hasChanges) {
         const c = await isolatedGit(['commit', '-m', report.subject, '-m', report.body]);
-        if (c.code !== 0) return `commit refused: ${lastLine(c.output)}`;
+        if (c.code !== 0) return `commit refused: ${gitReason(c.output)}`;
         const reset = await git(['reset', '--quiet', 'HEAD', '--', ...paths], top);
-        if (reset.code !== 0) return `committed; could not refresh staged paths: ${lastLine(reset.output)}`;
+        if (reset.code !== 0) return `committed; could not refresh staged paths: ${gitReason(reset.output)}`;
       }
       const upstream = await git(['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], top);
       if (upstream.truncated) return `${hasChanges ? 'committed; ' : ''}could not inspect upstream: Git output was truncated; deliver manually`;
       if (upstream.code !== 0) return hasChanges ? 'committed; no upstream to push to' : 'no upstream to push to';
       const pushed = await git(['push'], top);
-      if (pushed.code !== 0) return hasChanges ? `committed; push failed: ${lastLine(pushed.output)}` : `push failed: ${lastLine(pushed.output)}`;
+      if (pushed.code !== 0) return hasChanges ? `committed; push failed: ${gitReason(pushed.output)}` : `push failed: ${gitReason(pushed.output)}`;
       return hasChanges ? 'committed and pushed' : 'pushed';
     } finally {
       fs.rmSync(scratch, { recursive: true, force: true });
@@ -290,7 +295,7 @@ export function createAutoCollect({
     if (!inside(canonical(top), physical)) return { problem: 'the approved topics folder resolves outside the repository' };
     const gitDir = await git(['rev-parse', '--absolute-git-dir'], top);
     if (gitDir.truncated) return { problem: 'could not inspect Git metadata: Git output was truncated; inspect the target manually' };
-    if (gitDir.code !== 0) return { problem: `could not inspect Git metadata: ${lastLine(gitDir.output)}` };
+    if (gitDir.code !== 0) return { problem: `could not inspect Git metadata: ${gitReason(gitDir.output)}` };
     if (gitDir.code === 0 && inside(canonical(gitDir.output.trim()), physical)) {
       return { problem: 'new topics cannot be created in Git metadata' };
     }
@@ -327,12 +332,12 @@ export function createAutoCollect({
     const pathspecs = [...paths, `:(exclude)${excludedPath}`];
     const status = await git(['status', '--porcelain=v1', '-z', '--untracked-files=all', '--', ...pathspecs], top);
     if (status.truncated) throw new Error('could not inspect pending delivery: Git output was truncated; deliver manually');
-    if (status.code !== 0) throw new Error(`could not inspect pending delivery: ${lastLine(status.output)}`);
+    if (status.code !== 0) throw new Error(`could not inspect pending delivery: ${gitReason(status.output)}`);
     const hash = crypto.createHash('sha256');
     hash.update(status.output);
     const diff = await git(['diff', '--binary', 'HEAD', '--', ...pathspecs], top);
     if (diff.truncated) throw new Error('could not fingerprint pending delivery: Git output was truncated; deliver manually');
-    if (diff.code !== 0) throw new Error(`could not fingerprint pending delivery: ${lastLine(diff.output)}`);
+    if (diff.code !== 0) throw new Error(`could not fingerprint pending delivery: ${gitReason(diff.output)}`);
     hash.update(diff.output);
     for (const record of status.output.split('\0')) {
       if (!record.startsWith('?? ')) continue;
@@ -563,7 +568,7 @@ export function createAutoCollect({
       const top = await repoTop(dir);
       if (!top) { note(`${dir} is not in a git repository; requests travel by git`); return status(); }
       const pulled = await git(['pull', '--ff-only'], top);
-      if (pulled.code !== 0) note(`git pull did not run cleanly (${lastLine(pulled.output)}); collection blocked until synchronization succeeds`);
+      if (pulled.code !== 0) note(`git pull did not run cleanly (${gitReason(pulled.output)}); collection blocked until synchronization succeeds`);
       const requests = pulled.code === 0 ? listRequests(dir) : [];
 
       for (const item of requests) {

research-kit/lib/corpus.mjs

@@ -495,7 +495,9 @@ export function readLedger(root) {
   const file = resolve(root, PATHS.ledger);
   const text = readText(file);
   if (text === null) return { present: false, entries: [], problems: [], lines: [] };
-  const lines = text.split(/\r?\n/);
+  // An editor's leading BOM is outside the entries the chain hashes. Ignore it while
+  // reading, as the JSON and Markdown readers do; never rewrite the evidence file.
+  const lines = text.replace(/^\uFEFF/, '').split(/\r?\n/);
   const entries = [];
   const problems = [];
   lines.forEach((line, idx) => {

research-kit/lib/decompose.mjs

@@ -28,14 +28,19 @@ export function parseRecipe(text) {
   let name = '';
   if (match) {
     let inList = false;
-    for (const line of match[1].split(/\r?\n/)) {
+    for (const [offset, line] of match[1].split(/\r?\n/).entries()) {
       const named = line.match(/^name:\s*(.*)$/);
       if (named) { name = named[1].trim(); continue; }
       if (/^dimensions:\s*$/.test(line)) { inList = true; continue; }
       if (inList) {
         const item = line.match(/^\s*-\s+(.*)$/);
         if (!item) { if (line.trim()) inList = false; continue; }
         const [label, why] = item[1].split(/\s*\|\s*/);
+        if (!label.trim()) {
+          const err = new Error(`line ${offset + 2}: the dimension label is blank`);
+          err.code = 'INVALID_RECIPE';
+          throw err;
+        }
         dimensions.push({ name: label.trim(), why: (why ?? '').trim() });
       }
     }
@@ -54,7 +59,17 @@ export function loadRecipe(nameOrPath) {
     err.code = 'UNKNOWN_RECIPE';
     throw err;
   }
-  return { ...parseRecipe(text), file: direct };
+  let parsed;
+  try { parsed = parseRecipe(text); } catch (err) {
+    if (err.code === 'INVALID_RECIPE') err.message = `recipe "${direct}" ${err.message}`;
+    throw err;
+  }
+  if (!parsed.dimensions.length) {
+    const err = new Error(`recipe "${direct}" holds no dimensions - use a front-matter dimensions list of "- label | why" lines; see ${RECIPE_DIR}`);
+    err.code = 'INVALID_RECIPE';
+    throw err;
+  }
+  return { ...parsed, file: direct };
 }
 
 /** Where the facts probably live: the official hosts a search keeps pointing at. */

research-kit/lib/finding.mjs

@@ -331,6 +331,9 @@ export function findingWithContext(markdown, fallback = '') {
 
   for (let i = 0; i < lines.length; i += 1) {
     const raw = lines[i];
+    // Excerpts use physical paragraph boundaries, including ones inside skipped fences.
+    // Remember the start while scanning so recovery never inserts each line at the front.
+    if (!raw.trim() || /^\s{0,3}#{1,6}\s/.test(raw)) blockStart = i + 1;
     if (/^\s*```/.test(raw)) { inFence = !inFence; continue; }
     if (inFence) continue;
 
@@ -363,15 +366,15 @@ export function findingWithContext(markdown, fallback = '') {
     // An FAQ question is usually plain text, not a heading, so the heading rules cannot
     // see it. Remember whether the previous non-empty line asked something.
     const answersQuestion = lastLine.endsWith('?');
-    if (line) { if (!block.length) blockStart = i; block.push(line); lastLine = line; }
+    if (line) { block.push(line); lastLine = line; }
     if (isNoise(line)) continue;
     for (const sentence of sentences(line)) {
       const text = sentence.trim();
       if (text.length < 25) continue;
       if (isNoise(text)) continue;
       // The excerpt is resolved after the paragraph closes, so a sentence carries the
       // whole paragraph it came from rather than just itself.
-      candidates.push({ text, fromTable: false, heading, headings: stack.filter(Boolean).join(' / '), answersQuestion, lineIndex: i });
+      candidates.push({ text, fromTable: false, heading, headings: stack.filter(Boolean).join(' / '), answersQuestion, lineIndex: i, blockStart });
     }
   }
   flushBlock();
@@ -396,10 +399,13 @@ export function findingWithContext(markdown, fallback = '') {
   // Recover the paragraph the winner sat in, bounded. A table row already has one.
   let excerpt = best.excerpt ?? '';
   if (!excerpt && best.lineIndex !== undefined) {
-    const para = [];
-    for (let i = best.lineIndex; i >= 0 && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i -= 1) para.unshift(stripMarkdown(lines[i]));
-    for (let i = best.lineIndex + 1; i < lines.length && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i += 1) para.push(stripMarkdown(lines[i]));
-    excerpt = para.filter(Boolean).join(' ').trim();
+    for (let i = best.blockStart; i < lines.length && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i += 1) {
+      const line = stripMarkdown(lines[i]);
+      if (!line) continue;
+      excerpt += `${excerpt ? ' ' : ''}${line}`;
+      // One character beyond the cap proves truncation, including an exact-cap prefix.
+      if (excerpt.length > EXCERPT_CHARS) break;
+    }
   }
   if (excerpt.length > EXCERPT_CHARS) excerpt = `${excerpt.slice(0, EXCERPT_CHARS).trimEnd()}…`;