diff --git a/apps/api/src/__tests__/env-documented.test.ts b/apps/api/src/__tests__/env-documented.test.ts new file mode 100644 index 0000000..a454ed7 --- /dev/null +++ b/apps/api/src/__tests__/env-documented.test.ts @@ -0,0 +1,82 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { describe, expect, it } from 'vitest' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..') + +const walk = (dir: string, out: string[] = []): string[] => { + for (const entry of readdirSync(dir)) { + if (entry === 'node_modules' || entry === 'dist' || entry === '.next' || entry === '.turbo') { + continue + } + const full = join(dir, entry) + if (statSync(full).isDirectory()) walk(full, out) + else if (/\.(ts|tsx|mjs)$/.test(entry) && !full.includes('__tests__')) out.push(full) + } + return out +} + +/** + * A knob nobody can find is a knob that does not exist. Three of these — the queue + * visibility timeout and the branch and shard overrides — worked perfectly and were + * discoverable only by reading source, which is how an operator ends up rebuilding + * behaviour the platform already has. + */ +const DOCUMENTATION = [ + 'docs/configuration.md', + 'docs/architecture.md', + 'docs/threat-model.md', + 'README.md', +].map((file) => { + try { + return readFileSync(join(root, file), 'utf8') + } catch { + return '' + } +}) + +const ACTIONS = ['flakemetry', 'flakemetry-pr-comment', 'flakemetry-gate'].map((name) => { + try { + return readFileSync(join(root, '.github/actions', name, 'action.yml'), 'utf8') + } catch { + return '' + } +}) + +/** + * Named values rather than knobs, or internal plumbing behind a documented command. Each + * is here on purpose — the point of the check is that silence has to be chosen. + */ +const NOT_A_KNOB: Readonly> = { + FLAKEMETRY_TOKEN: 'the credential itself, documented everywhere it is used', + FLAKEMETRY_ENDPOINT: 'the instance URL, same', + FLAKEMETRY_SEED_FORCE: 'internal to the demo seed, whose documented interface is `pnpm demo`', +} + +describe('every FLAKEMETRY_ variable the code reads is documented', () => { + const found = new Set() + for (const file of [...walk(join(root, 'apps')), ...walk(join(root, 'packages'))]) { + const source = readFileSync(file, 'utf8') + for (const match of source.matchAll(/\bFLAKEMETRY_[A-Z0-9_]+/g)) found.add(match[0]) + } + + it('reads the code it is meant to be checking', () => { + // Guard the guard: an empty scan documents nothing and complains about nothing. + expect(found.size).toBeGreaterThan(20) + expect(found).toContain('FLAKEMETRY_QUEUE_VISIBILITY_MS') + }) + + it('finds each one in the documentation or an action definition', () => { + const undocumented = [...found] + .filter((name) => !(name in NOT_A_KNOB)) + .filter((name) => ![...DOCUMENTATION, ...ACTIONS].some((text) => text.includes(name))) + .sort() + + expect( + undocumented, + 'document these in docs/configuration.md — a knob nobody can find is a knob that does not exist', + ).toEqual([]) + }) +}) diff --git a/docs/configuration.md b/docs/configuration.md index 762d3dc..12ba1e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,6 +91,9 @@ Unknown keys are rejected with an error naming the offending path — typos fail | `FLAKEMETRY_BUFFER_DIR` | Directory to buffer runs to when delivery fails; replayed on the next run | | `FLAKEMETRY_SAMPLE_RATE` | Fraction (0–1) of **passing** runs to deliver; runs containing a failure or flake are always delivered | | `FLAKEMETRY_COMPRESSION` | `gzip` to compress OTLP export (the ingestion API decompresses gzip request bodies) | +| `FLAKEMETRY_COMMIT_SHA` | Commit the run belongs to, when CI detection cannot find it — a run without one lands on `0000000`, and every run on that placeholder looks like the same commit to the scorer | +| `FLAKEMETRY_BRANCH` | Branch the run belongs to, same case; the fallback is `local` | +| `FLAKEMETRY_SHARD_INDEX` / `FLAKEMETRY_SHARD_TOTAL` | Shard position when the runner shards in a way detection does not recognise. Both are needed; a total of 1 is treated as unsharded | | `FLAKEMETRY_CODEOWNERS_FILE` | Explicit path to a CODEOWNERS file to sync; otherwise the reporter looks for `CODEOWNERS`, `.github/CODEOWNERS`, or `docs/CODEOWNERS` walking up from the test root | | `FLAKEMETRY_IDEMPOTENCY_KEY` | Explicit idempotency key for the run; makes re-delivery safe. Defaults to the run span trace id. Sharded runs get a per-shard `-shard` suffix automatically | @@ -115,9 +118,27 @@ Rate-limit and backpressure state are held per API process (in-memory). Running | `POLL_INTERVAL_MS` | Idle poll interval between dequeue attempts | | `FLAKEMETRY_SELF_OTEL_ENDPOINT` | OTLP endpoint for the worker's own metrics (processing lag, throughput, error rate, queue depth) | | `FLAKEMETRY_CLUSTER_THRESHOLD` | Jaccard similarity (0–1) above which a new error signature joins an existing cluster (default `0.5`) | +| `FLAKEMETRY_QUEUE_VISIBILITY_MS` | How long a dequeued job stays invisible to other workers before it is redelivered (default `300000`). Raise it only if a single run legitimately takes longer than this to process — lowering it below the slowest job causes the same run to be processed twice | +| `FLAKEMETRY_EXECUTION_RETENTION_DAYS` / `FLAKEMETRY_ARTIFACT_RETENTION_DAYS` | Global retention floor for projects with no per-project policy; see [Trend rollups and retention](#trend-rollups-and-retention) | The worker emits domain events (`run.processed`, `identity.created`, `identity.moved`, `score.updated`, `flaky.detected`, `quarantine.changed`, `suite.regressed`, `suite.slowed`, `rca.created`) after each job commits — the seam downstream stages such as signature clustering, AI RCA, and notifications subscribe to. +### AI root-cause analysis + +`ai.rca` turns the feature on; these decide what it talks to. Setting `FLAKEMETRY_AI_RCA=true` +without a provider gets you nothing — the worker has nothing to ask. + +| Variable | Effect | +|---|---| +| `FLAKEMETRY_AI_PROVIDER` | `anthropic` or `ollama`. Unset means RCA stays off however `ai.rca` is set | +| `FLAKEMETRY_AI_API_KEY` | Provider credentials. `ANTHROPIC_API_KEY` is also read, so an existing environment works unchanged | +| `FLAKEMETRY_AI_MODEL` | Model id; each provider has a sensible default | +| `FLAKEMETRY_AI_ENDPOINT` | Base URL, for a self-hosted Ollama or a proxy | +| `FLAKEMETRY_AI_TIMEOUT_MS` | Per-request ceiling. RCA is best-effort — a slow provider must not hold up processing | + +Spend is bounded by `ai.dailyTokenBudget` per project, and only genuinely new error +signatures reach the model at all; the rest are answered from the cluster's cached analysis. + ### Notifications The worker pushes intelligence to Slack, Discord and email. Webhook delivery is best-effort and de-duplicated per channel so a flapping test can't spam a channel. Channels come from two places, applied together: **global env channels** (below) and **per-project channels** configured in **Settings → Notifications** (add a Slack/Discord webhook or an email address with an event filter). Events: `flaky_detected`, `quarantine_changed`, `rca_ready`, `suite_regressed` (a suite's fail-rate crossing its trailing baseline), and `suite_slowed` (a suite's average duration rising well above its trailing baseline).