docs: document the settings that only existed in source

apps/api/src/__tests__/env-documented.test.ts

@@ -0,0 +1,82 @@
+import { readdirSync, readFileSync, statSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { describe, expect, it } from 'vitest'
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..')
+
+const walk = (dir: string, out: string[] = []): string[] => {
+  for (const entry of readdirSync(dir)) {
+    if (entry === 'node_modules' || entry === 'dist' || entry === '.next' || entry === '.turbo') {
+      continue
+    }
+    const full = join(dir, entry)
+    if (statSync(full).isDirectory()) walk(full, out)
+    else if (/\.(ts|tsx|mjs)$/.test(entry) && !full.includes('__tests__')) out.push(full)
+  }
+  return out
+}
+
+/**
+ * A knob nobody can find is a knob that does not exist. Three of these — the queue
+ * visibility timeout and the branch and shard overrides — worked perfectly and were
+ * discoverable only by reading source, which is how an operator ends up rebuilding
+ * behaviour the platform already has.
+ */
+const DOCUMENTATION = [
+  'docs/configuration.md',
+  'docs/architecture.md',
+  'docs/threat-model.md',
+  'README.md',
+].map((file) => {
+  try {
+    return readFileSync(join(root, file), 'utf8')
+  } catch {
+    return ''
+  }
+})
+
+const ACTIONS = ['flakemetry', 'flakemetry-pr-comment', 'flakemetry-gate'].map((name) => {
+  try {
+    return readFileSync(join(root, '.github/actions', name, 'action.yml'), 'utf8')
+  } catch {
+    return ''
+  }
+})
+
+/**
+ * Named values rather than knobs, or internal plumbing behind a documented command. Each
+ * is here on purpose — the point of the check is that silence has to be chosen.
+ */
+const NOT_A_KNOB: Readonly<Record<string, string>> = {
+  FLAKEMETRY_TOKEN: 'the credential itself, documented everywhere it is used',
+  FLAKEMETRY_ENDPOINT: 'the instance URL, same',
+  FLAKEMETRY_SEED_FORCE: 'internal to the demo seed, whose documented interface is `pnpm demo`',
+}
+
+describe('every FLAKEMETRY_ variable the code reads is documented', () => {
+  const found = new Set<string>()
+  for (const file of [...walk(join(root, 'apps')), ...walk(join(root, 'packages'))]) {
+    const source = readFileSync(file, 'utf8')
+    for (const match of source.matchAll(/\bFLAKEMETRY_[A-Z0-9_]+/g)) found.add(match[0])
+  }
+
+  it('reads the code it is meant to be checking', () => {
+    // Guard the guard: an empty scan documents nothing and complains about nothing.
+    expect(found.size).toBeGreaterThan(20)
+    expect(found).toContain('FLAKEMETRY_QUEUE_VISIBILITY_MS')
+  })
+
+  it('finds each one in the documentation or an action definition', () => {
+    const undocumented = [...found]
+      .filter((name) => !(name in NOT_A_KNOB))
+      .filter((name) => ![...DOCUMENTATION, ...ACTIONS].some((text) => text.includes(name)))
+      .sort()
+
+    expect(
+      undocumented,
+      'document these in docs/configuration.md — a knob nobody can find is a knob that does not exist',
+    ).toEqual([])
+  })
+})

docs/configuration.md

@@ -91,6 +91,9 @@ Unknown keys are rejected with an error naming the offending path — typos fail
 | `FLAKEMETRY_BUFFER_DIR` | Directory to buffer runs to when delivery fails; replayed on the next run |
 | `FLAKEMETRY_SAMPLE_RATE` | Fraction (0–1) of **passing** runs to deliver; runs containing a failure or flake are always delivered |
 | `FLAKEMETRY_COMPRESSION` | `gzip` to compress OTLP export (the ingestion API decompresses gzip request bodies) |
+| `FLAKEMETRY_COMMIT_SHA` | Commit the run belongs to, when CI detection cannot find it — a run without one lands on `0000000`, and every run on that placeholder looks like the same commit to the scorer |
+| `FLAKEMETRY_BRANCH` | Branch the run belongs to, same case; the fallback is `local` |
+| `FLAKEMETRY_SHARD_INDEX` / `FLAKEMETRY_SHARD_TOTAL` | Shard position when the runner shards in a way detection does not recognise. Both are needed; a total of 1 is treated as unsharded |
 | `FLAKEMETRY_CODEOWNERS_FILE` | Explicit path to a CODEOWNERS file to sync; otherwise the reporter looks for `CODEOWNERS`, `.github/CODEOWNERS`, or `docs/CODEOWNERS` walking up from the test root |
 | `FLAKEMETRY_IDEMPOTENCY_KEY` | Explicit idempotency key for the run; makes re-delivery safe. Defaults to the run span trace id. Sharded runs get a per-shard `-shard<index>` suffix automatically |
 
@@ -115,9 +118,27 @@ Rate-limit and backpressure state are held per API process (in-memory). Running
 | `POLL_INTERVAL_MS` | Idle poll interval between dequeue attempts |
 | `FLAKEMETRY_SELF_OTEL_ENDPOINT` | OTLP endpoint for the worker's own metrics (processing lag, throughput, error rate, queue depth) |
 | `FLAKEMETRY_CLUSTER_THRESHOLD` | Jaccard similarity (0–1) above which a new error signature joins an existing cluster (default `0.5`) |
+| `FLAKEMETRY_QUEUE_VISIBILITY_MS` | How long a dequeued job stays invisible to other workers before it is redelivered (default `300000`). Raise it only if a single run legitimately takes longer than this to process — lowering it below the slowest job causes the same run to be processed twice |
+| `FLAKEMETRY_EXECUTION_RETENTION_DAYS` / `FLAKEMETRY_ARTIFACT_RETENTION_DAYS` | Global retention floor for projects with no per-project policy; see [Trend rollups and retention](#trend-rollups-and-retention) |
 
 The worker emits domain events (`run.processed`, `identity.created`, `identity.moved`, `score.updated`, `flaky.detected`, `quarantine.changed`, `suite.regressed`, `suite.slowed`, `rca.created`) after each job commits — the seam downstream stages such as signature clustering, AI RCA, and notifications subscribe to.
 
+### AI root-cause analysis
+
+`ai.rca` turns the feature on; these decide what it talks to. Setting `FLAKEMETRY_AI_RCA=true`
+without a provider gets you nothing — the worker has nothing to ask.
+
+| Variable | Effect |
+|---|---|
+| `FLAKEMETRY_AI_PROVIDER` | `anthropic` or `ollama`. Unset means RCA stays off however `ai.rca` is set |
+| `FLAKEMETRY_AI_API_KEY` | Provider credentials. `ANTHROPIC_API_KEY` is also read, so an existing environment works unchanged |
+| `FLAKEMETRY_AI_MODEL` | Model id; each provider has a sensible default |
+| `FLAKEMETRY_AI_ENDPOINT` | Base URL, for a self-hosted Ollama or a proxy |
+| `FLAKEMETRY_AI_TIMEOUT_MS` | Per-request ceiling. RCA is best-effort — a slow provider must not hold up processing |
+
+Spend is bounded by `ai.dailyTokenBudget` per project, and only genuinely new error
+signatures reach the model at all; the rest are answered from the cluster's cached analysis.
+
 ### Notifications
 
 The worker pushes intelligence to Slack, Discord and email. Webhook delivery is best-effort and de-duplicated per channel so a flapping test can't spam a channel. Channels come from two places, applied together: **global env channels** (below) and **per-project channels** configured in **Settings → Notifications** (add a Slack/Discord webhook or an email address with an event filter). Events: `flaky_detected`, `quarantine_changed`, `rca_ready`, `suite_regressed` (a suite's fail-rate crossing its trailing baseline), and `suite_slowed` (a suite's average duration rising well above its trailing baseline).