From 528160f6224e35e68112aae80b9d768025c7091b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 16:15:49 +0000 Subject: [PATCH] fix(tooling): derive the docs-audit scope from the filesystem instead of a hand-kept list (#4851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.claude/workflows/docs-accuracy-audit.js` carried its default audit scope inline, as a hand-kept `ALL_HANDWRITTEN` array behind a "keep in sync with `affected-docs.mjs --all`" comment. Nothing checked that promise, and it had rotted in BOTH directions: - 16 listed paths no longer existed — 10 of them the whole `content/docs/protocol/objectos/**` directory, renamed to `protocol/kernel/`. An audit agent pointed at a non-existent file reads nothing and reports `fixCount: 0`, indistinguishable in the run summary from a doc that was checked and found accurate. That is how #4781 and #4817 sat in `protocol/kernel/` for ~2 months under green "full" audits. - 48 existing docs were absent from the list — including all 9 of `protocol/kernel/**` and the whole `content/docs/capabilities/` directory. A run logging `FULL audit` was auditing 130 of 178 hand-written docs. The list stays inline because it must: a workflow script runs in a `node:vm` context with no require/import/filesystem, so it can neither walk `content/docs/` nor read a JSON artifact. So it is GENERATED instead — `scripts/docs-audit/check-audit-scope.mjs --write` derives it from `affected-docs.mjs --all` (one definition of "hand-written doc"), and `pnpm check:docs-audit-scope` fails in lint.yml when the block and `content/docs/` disagree in either direction, naming every entry. Missing markers or an unparseable block fail too — a gate that cannot find its subject must go red, not green. A CI gate only sees the default list, so two more nets cover the rest: - the workflow preflights its resolved scope (including a caller-supplied `args.docs`) and refuses to start, naming every path that does not exist; the preflight's own arithmetic is reconciled against the scope, so a verdict that cannot account for every path exactly once is a failed preflight, not a pass; - every audit agent reports `docExists` from the path that actually opens the file, and the run throws if any comes back false — a self-check that runs somewhere other than the real read path proves nothing about it (#4868). Same discipline as #4690 / #4777 / #4804 / #4835 / #4868 / #4890. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018iARDqtrhQgz6fVHDeDkbQ --- .changeset/docs-audit-scope-derived.md | 47 +++ .claude/workflows/docs-accuracy-audit.js | 320 ++++++++++++++++++++- .github/workflows/lint.yml | 14 + package.json | 1 + scripts/docs-audit/README.md | 46 ++- scripts/docs-audit/affected-docs.mjs | 2 +- scripts/docs-audit/check-audit-scope.mjs | 349 +++++++++++++++++++++++ 7 files changed, 769 insertions(+), 10 deletions(-) create mode 100644 .changeset/docs-audit-scope-derived.md create mode 100644 scripts/docs-audit/check-audit-scope.mjs diff --git a/.changeset/docs-audit-scope-derived.md b/.changeset/docs-audit-scope-derived.md new file mode 100644 index 0000000000..b4972dfcdb --- /dev/null +++ b/.changeset/docs-audit-scope-derived.md @@ -0,0 +1,47 @@ +--- +--- + +tooling: derive the docs-accuracy audit's scope from the filesystem and fail loudly when +it drifts (#4851) + +Release-nothing: touches `.claude/workflows/docs-accuracy-audit.js`, +`scripts/docs-audit/`, the root `check:docs-audit-scope` script and one `lint.yml` step — +no package code. + +`.claude/workflows/docs-accuracy-audit.js` carried its default audit scope inline, as a +hand-kept `ALL_HANDWRITTEN` array behind a "keep in sync with `affected-docs.mjs --all`" +comment. Nothing checked that promise, and it had rotted in **both** directions: + +- **16 listed paths no longer existed** — 10 of them the whole + `content/docs/protocol/objectos/**` directory, renamed to `protocol/kernel/`. A doc + path that resolves to nothing produces an audit agent that reads nothing and reports + `fixCount: 0`, which in the run summary is indistinguishable from a doc that was + checked and found accurate. That is how the accuracy defects in #4781 + (`runtime-capabilities.mdx` documenting a schema deleted in #3605) and #4817 + (`http-protocol.mdx` attributing the dispatcher's response shape to + `/api/v1/discovery`) survived ~2 months of green "full" audits. +- **48 existing docs were absent from it** — including all 9 of `protocol/kernel/**` and + the entire `content/docs/capabilities/` directory. A run logging + `FULL audit (no args.docs given)` was auditing 130 of 178 hand-written docs. + +The list stays inline because it must: a workflow script runs in a `node:vm` context +with no `require`, no `import` and no filesystem, so it can neither walk `content/docs/` +nor read a JSON artifact. So it is now **generated** rather than hand-kept — +`node scripts/docs-audit/check-audit-scope.mjs --write` derives it from +`affected-docs.mjs --all` (one definition of "hand-written doc", not a second walk to +drift), and `pnpm check:docs-audit-scope` fails in `lint.yml` when the block and +`content/docs/` disagree in either direction, naming every entry. + +Two more nets, because a CI gate can only see the *default* list: + +- the workflow **preflights its resolved scope** — including a caller-supplied + `args.docs` — and refuses to start, naming every path that does not exist. Its + arithmetic is reconciled against the scope, so a preflight that cannot account for + every path exactly once is a failed preflight, not a pass; +- every audit agent now reports `docExists` from the path that actually opens the file, + and the run **throws** if any comes back false. The preflight is not the real read + path, and a self-check that runs somewhere other than the real path proves nothing + about it (#4868). + +Same discipline as #4690 / #4777 / #4804 / #4835 / #4868 / #4890: a check whose subject +has gone missing must go red, never green-by-vacancy. diff --git a/.claude/workflows/docs-accuracy-audit.js b/.claude/workflows/docs-accuracy-audit.js index 2e15285b6b..ed2bd01497 100644 --- a/.claude/workflows/docs-accuracy-audit.js +++ b/.claude/workflows/docs-accuracy-audit.js @@ -3,15 +3,216 @@ export const meta = { description: 'Audit + fix hand-written ObjectStack docs against actual implementation, with adversarial verification. Scope with args.docs; defaults to all hand-written docs.', whenToUse: 'Periodic or change-scoped documentation accuracy verification. Pass args.docs = [paths] to scope (e.g. output of scripts/docs-audit/affected-docs.mjs); omit for a full audit of every hand-written doc.', phases: [ + { title: 'Scope Preflight', detail: 'resolve every doc path on disk; abort naming any that does not exist' }, { title: 'Audit & Fix', detail: 'one agent per doc: read, locate implementation, apply evidence-backed edits' }, { title: 'Adversarial Verify', detail: 'second agent re-checks each applied fix against code, repairs regressions' }, ], } -// Default scope = every hand-written doc (content/docs/** minus references/). Keep in -// sync with `node scripts/docs-audit/affected-docs.mjs --all`. Callers normally pass a -// scoped subset via args.docs (e.g. only docs whose backing packages/ code changed). -const ALL_HANDWRITTEN = ["content/docs/ai/actions-as-tools.mdx","content/docs/ai/agents.mdx","content/docs/ai/chatbot-integration.mdx","content/docs/ai/index.mdx","content/docs/ai/knowledge-rag.mdx","content/docs/ai/natural-language-queries.mdx","content/docs/ai/skills-reference.mdx","content/docs/ai/skills.mdx","content/docs/api/client-sdk.mdx","content/docs/api/data-api.mdx","content/docs/api/data-flow.mdx","content/docs/api/environment-routing.mdx","content/docs/api/error-catalog.mdx","content/docs/api/error-handling-client.mdx","content/docs/api/error-handling-server.mdx","content/docs/api/index.mdx","content/docs/api/metadata-api.mdx","content/docs/api/plugin-endpoints.mdx","content/docs/api/wire-format.mdx","content/docs/automation/approvals.mdx","content/docs/automation/flows.mdx","content/docs/automation/hook-bodies.mdx","content/docs/automation/hooks.mdx","content/docs/automation/index.mdx","content/docs/automation/webhooks.mdx","content/docs/automation/workflows.mdx","content/docs/concepts/architecture.mdx","content/docs/concepts/design-principles.mdx","content/docs/concepts/index.mdx","content/docs/concepts/metadata-driven.mdx","content/docs/concepts/metadata-lifecycle.mdx","content/docs/concepts/north-star.mdx","content/docs/data-modeling/analytics.mdx","content/docs/data-modeling/drivers.mdx","content/docs/data-modeling/external-datasources.mdx","content/docs/data-modeling/field-type-decision-tree.mdx","content/docs/data-modeling/field-types.mdx","content/docs/data-modeling/fields.mdx","content/docs/data-modeling/formulas.mdx","content/docs/data-modeling/index.mdx","content/docs/data-modeling/indexing.mdx","content/docs/data-modeling/objects.mdx","content/docs/data-modeling/queries.mdx","content/docs/data-modeling/relationships.mdx","content/docs/data-modeling/schema-design.mdx","content/docs/data-modeling/seed-data.mdx","content/docs/data-modeling/validation-rules.mdx","content/docs/data-modeling/validation.mdx","content/docs/deployment/cloud-artifact-api.mdx","content/docs/deployment/environment-variables.mdx","content/docs/deployment/index.mdx","content/docs/deployment/migration-from-objectql.mdx","content/docs/deployment/production-readiness.mdx","content/docs/deployment/publish-and-preview.mdx","content/docs/deployment/single-project-mode.mdx","content/docs/deployment/troubleshooting.mdx","content/docs/deployment/vercel.mdx","content/docs/getting-started/cli.mdx","content/docs/getting-started/common-patterns.mdx","content/docs/getting-started/examples.mdx","content/docs/getting-started/glossary.mdx","content/docs/getting-started/index.mdx","content/docs/getting-started/quick-reference.mdx","content/docs/getting-started/quick-start.mdx","content/docs/getting-started/validating-metadata.mdx","content/docs/index.mdx","content/docs/kernel/architecture.mdx","content/docs/kernel/cluster.mdx","content/docs/kernel/contracts/auth-service.mdx","content/docs/kernel/contracts/cache-service.mdx","content/docs/kernel/contracts/data-engine.mdx","content/docs/kernel/contracts/index.mdx","content/docs/kernel/contracts/metadata-service.mdx","content/docs/kernel/contracts/storage-service.mdx","content/docs/kernel/events.mdx","content/docs/kernel/index.mdx","content/docs/kernel/runtime-services/audit-service.mdx","content/docs/kernel/runtime-services/data-service.mdx","content/docs/kernel/runtime-services/email-service.mdx","content/docs/kernel/runtime-services/examples.mdx","content/docs/kernel/runtime-services/index.mdx","content/docs/kernel/runtime-services/queue-service.mdx","content/docs/kernel/runtime-services/settings-service.mdx","content/docs/kernel/runtime-services/sharing-service.mdx","content/docs/kernel/runtime-services/storage-service.mdx","content/docs/kernel/runtime-services/versioning.mdx","content/docs/kernel/services-checklist.mdx","content/docs/kernel/services.mdx","content/docs/permissions/access-recipes.mdx","content/docs/permissions/authentication.mdx","content/docs/permissions/authorization.mdx","content/docs/permissions/field-level-security.mdx","content/docs/permissions/index.mdx","content/docs/permissions/permission-metadata.mdx","content/docs/permissions/permission-sets.mdx","content/docs/permissions/permissions-matrix.mdx","content/docs/permissions/profiles.mdx","content/docs/permissions/roles.mdx","content/docs/permissions/sharing-rules.mdx","content/docs/permissions/sso.mdx","content/docs/plugins/adding-a-metadata-type.mdx","content/docs/plugins/anatomy.mdx","content/docs/plugins/development.mdx","content/docs/plugins/index.mdx","content/docs/plugins/packages.mdx","content/docs/protocol/backward-compatibility.mdx","content/docs/protocol/diagram.mdx","content/docs/protocol/index.mdx","content/docs/protocol/knowledge.mdx","content/docs/protocol/objectos/config-resolution.mdx","content/docs/protocol/objectos/error-handling.mdx","content/docs/protocol/objectos/http-protocol.mdx","content/docs/protocol/objectos/i18n-standard.mdx","content/docs/protocol/objectos/index.mdx","content/docs/protocol/objectos/lifecycle.mdx","content/docs/protocol/objectos/metadata-service.mdx","content/docs/protocol/objectos/plugin-spec.mdx","content/docs/protocol/objectos/realtime-protocol.mdx","content/docs/protocol/objectos/runtime-capabilities.mdx","content/docs/protocol/objectql/index.mdx","content/docs/protocol/objectql/query-syntax.mdx","content/docs/protocol/objectql/schema.mdx","content/docs/protocol/objectql/security.mdx","content/docs/protocol/objectql/state-machine.mdx","content/docs/protocol/objectql/types.mdx","content/docs/protocol/objectui/actions.mdx","content/docs/protocol/objectui/concept.mdx","content/docs/protocol/objectui/index.mdx","content/docs/protocol/objectui/layout-dsl.mdx","content/docs/protocol/objectui/record-alert.mdx","content/docs/protocol/objectui/widget-contract.mdx","content/docs/releases/implementation-status.mdx","content/docs/releases/index.mdx","content/docs/releases/v9.mdx","content/docs/ui/apps.mdx","content/docs/ui/create-vs-edit-form.mdx","content/docs/ui/dashboards.mdx","content/docs/ui/doc-pages.mdx","content/docs/ui/field-grouping-and-order.mdx","content/docs/ui/forms.mdx","content/docs/ui/index.mdx","content/docs/ui/pages.mdx","content/docs/ui/public-data-collection.mdx","content/docs/ui/role-based-interfaces.mdx","content/docs/ui/setup-app.mdx","content/docs/ui/views.mdx"] +// Default scope = every hand-written doc (content/docs/** minus references/). Callers +// normally pass a scoped subset via args.docs (e.g. only docs whose backing packages/ +// code changed). +// +// The list is inline because it has to be: a workflow script runs in a `node:vm` +// context whose only globals are log/phase/console/budget/timers plus agent/parallel/ +// pipeline/workflow/args, with code generation disabled — no require, no import, no +// filesystem. It cannot enumerate content/docs/ itself, nor read a JSON artifact. +// +// So it is GENERATED instead: `node scripts/docs-audit/check-audit-scope.mjs --write` +// derives it from the filesystem, and the same script without --write is a CI gate +// (`pnpm check:docs-audit-scope`) that fails when the block and content/docs/ disagree +// in EITHER direction. It used to be hand-kept behind a "keep in sync" comment, and by +// #4851 had rotted both ways at once — 16 entries pointing at files that no longer +// existed (the whole renamed protocol/objectos → protocol/kernel directory among them) +// and 48 existing docs missing from it — with every full-audit run reporting green. +// +// GENERATED — do not hand-edit. `node scripts/docs-audit/check-audit-scope.mjs --write` +// derives this from the filesystem (every content/docs/**/*.mdx except +// references/, via `affected-docs.mjs --all`); the same script run without +// --write fails CI when the two disagree in either direction. See #4851: this +// list was hand-kept, 16 entries pointed at files that no longer existed and 48 +// existing docs were absent from it, and a "FULL audit" reported green over both. +const ALL_HANDWRITTEN = [ + "content/docs/ai/actions-as-tools.mdx", + "content/docs/ai/agents.mdx", + "content/docs/ai/connect-mcp.mdx", + "content/docs/ai/index.mdx", + "content/docs/ai/knowledge-rag.mdx", + "content/docs/ai/natural-language-queries.mdx", + "content/docs/ai/skills-reference.mdx", + "content/docs/ai/skills.mdx", + "content/docs/api/client-sdk.mdx", + "content/docs/api/data-api.mdx", + "content/docs/api/data-flow.mdx", + "content/docs/api/environment-routing.mdx", + "content/docs/api/error-catalog.mdx", + "content/docs/api/error-handling-client.mdx", + "content/docs/api/error-handling-server.mdx", + "content/docs/api/index.mdx", + "content/docs/api/metadata-api.mdx", + "content/docs/api/plugin-endpoints.mdx", + "content/docs/api/wire-format.mdx", + "content/docs/automation/approvals.mdx", + "content/docs/automation/connectors.mdx", + "content/docs/automation/flows.mdx", + "content/docs/automation/hook-bodies.mdx", + "content/docs/automation/hooks.mdx", + "content/docs/automation/index.mdx", + "content/docs/automation/webhooks.mdx", + "content/docs/automation/workflows.mdx", + "content/docs/build-without-code.mdx", + "content/docs/capabilities/ai.mdx", + "content/docs/capabilities/analytics.mdx", + "content/docs/capabilities/approvals.mdx", + "content/docs/capabilities/automation.mdx", + "content/docs/capabilities/data.mdx", + "content/docs/capabilities/forms.mdx", + "content/docs/capabilities/index.mdx", + "content/docs/capabilities/integrations.mdx", + "content/docs/capabilities/permissions.mdx", + "content/docs/capabilities/request-template.mdx", + "content/docs/capabilities/views.mdx", + "content/docs/concepts/architecture.mdx", + "content/docs/concepts/design-principles.mdx", + "content/docs/concepts/index.mdx", + "content/docs/concepts/metadata-driven.mdx", + "content/docs/concepts/metadata-lifecycle.mdx", + "content/docs/concepts/north-star.mdx", + "content/docs/data-modeling/analytics.mdx", + "content/docs/data-modeling/drivers.mdx", + "content/docs/data-modeling/external-datasources.mdx", + "content/docs/data-modeling/field-type-decision-tree.mdx", + "content/docs/data-modeling/field-types.mdx", + "content/docs/data-modeling/fields.mdx", + "content/docs/data-modeling/formulas.mdx", + "content/docs/data-modeling/index.mdx", + "content/docs/data-modeling/indexing.mdx", + "content/docs/data-modeling/objects.mdx", + "content/docs/data-modeling/queries.mdx", + "content/docs/data-modeling/relationships.mdx", + "content/docs/data-modeling/schema-design.mdx", + "content/docs/data-modeling/seed-data.mdx", + "content/docs/data-modeling/validation-rules.mdx", + "content/docs/data-modeling/validation.mdx", + "content/docs/deployment/backup-restore.mdx", + "content/docs/deployment/cli.mdx", + "content/docs/deployment/environment-variables.mdx", + "content/docs/deployment/index.mdx", + "content/docs/deployment/migration-from-objectql.mdx", + "content/docs/deployment/production-readiness.mdx", + "content/docs/deployment/publish-and-preview.mdx", + "content/docs/deployment/self-hosting.mdx", + "content/docs/deployment/single-project-mode.mdx", + "content/docs/deployment/tenancy-modes.mdx", + "content/docs/deployment/troubleshooting.mdx", + "content/docs/deployment/validating-metadata.mdx", + "content/docs/deployment/vercel.mdx", + "content/docs/getting-started/build-with-claude-code.mdx", + "content/docs/getting-started/common-patterns.mdx", + "content/docs/getting-started/examples.mdx", + "content/docs/getting-started/glossary.mdx", + "content/docs/getting-started/how-ai-development-works.mdx", + "content/docs/getting-started/index.mdx", + "content/docs/getting-started/quick-reference.mdx", + "content/docs/getting-started/quick-start.mdx", + "content/docs/getting-started/your-first-project.mdx", + "content/docs/index.mdx", + "content/docs/kernel/architecture.mdx", + "content/docs/kernel/cluster.mdx", + "content/docs/kernel/contracts/auth-service.mdx", + "content/docs/kernel/contracts/cache-service.mdx", + "content/docs/kernel/contracts/data-engine.mdx", + "content/docs/kernel/contracts/index.mdx", + "content/docs/kernel/contracts/metadata-service.mdx", + "content/docs/kernel/contracts/storage-service.mdx", + "content/docs/kernel/events.mdx", + "content/docs/kernel/index.mdx", + "content/docs/kernel/runtime-services/audit-service.mdx", + "content/docs/kernel/runtime-services/data-service.mdx", + "content/docs/kernel/runtime-services/email-service.mdx", + "content/docs/kernel/runtime-services/examples.mdx", + "content/docs/kernel/runtime-services/index.mdx", + "content/docs/kernel/runtime-services/queue-service.mdx", + "content/docs/kernel/runtime-services/settings-service.mdx", + "content/docs/kernel/runtime-services/sharing-service.mdx", + "content/docs/kernel/runtime-services/sms-service.mdx", + "content/docs/kernel/runtime-services/storage-service.mdx", + "content/docs/kernel/runtime-services/versioning.mdx", + "content/docs/kernel/services-checklist.mdx", + "content/docs/kernel/services.mdx", + "content/docs/permissions/access-matrix.mdx", + "content/docs/permissions/access-recipes.mdx", + "content/docs/permissions/administrator-guide.mdx", + "content/docs/permissions/attachments-access.mdx", + "content/docs/permissions/authentication.mdx", + "content/docs/permissions/authorization.mdx", + "content/docs/permissions/delegated-administration.mdx", + "content/docs/permissions/explain.mdx", + "content/docs/permissions/field-level-security.mdx", + "content/docs/permissions/index.mdx", + "content/docs/permissions/permission-metadata.mdx", + "content/docs/permissions/permission-sets.mdx", + "content/docs/permissions/permissions-matrix.mdx", + "content/docs/permissions/positions.mdx", + "content/docs/permissions/profiles.mdx", + "content/docs/permissions/rls.mdx", + "content/docs/permissions/sharing-rules.mdx", + "content/docs/permissions/sso.mdx", + "content/docs/plugins/adding-a-metadata-type.mdx", + "content/docs/plugins/anatomy.mdx", + "content/docs/plugins/development.mdx", + "content/docs/plugins/index.mdx", + "content/docs/plugins/packages.mdx", + "content/docs/protocol/backward-compatibility.mdx", + "content/docs/protocol/diagram.mdx", + "content/docs/protocol/index.mdx", + "content/docs/protocol/kernel/config-resolution.mdx", + "content/docs/protocol/kernel/error-handling.mdx", + "content/docs/protocol/kernel/http-protocol.mdx", + "content/docs/protocol/kernel/i18n-standard.mdx", + "content/docs/protocol/kernel/index.mdx", + "content/docs/protocol/kernel/lifecycle.mdx", + "content/docs/protocol/kernel/metadata-service.mdx", + "content/docs/protocol/kernel/plugin-spec.mdx", + "content/docs/protocol/kernel/realtime-protocol.mdx", + "content/docs/protocol/knowledge.mdx", + "content/docs/protocol/objectql/index.mdx", + "content/docs/protocol/objectql/query-syntax.mdx", + "content/docs/protocol/objectql/schema.mdx", + "content/docs/protocol/objectql/security.mdx", + "content/docs/protocol/objectql/state-machine.mdx", + "content/docs/protocol/objectql/types.mdx", + "content/docs/protocol/objectui/actions.mdx", + "content/docs/protocol/objectui/concept.mdx", + "content/docs/protocol/objectui/index.mdx", + "content/docs/protocol/objectui/layout-dsl.mdx", + "content/docs/protocol/objectui/record-alert.mdx", + "content/docs/protocol/objectui/widget-contract.mdx", + "content/docs/releases/implementation-status.mdx", + "content/docs/releases/index.mdx", + "content/docs/releases/v12.mdx", + "content/docs/releases/v13.mdx", + "content/docs/releases/v14.mdx", + "content/docs/releases/v15.mdx", + "content/docs/releases/v16.mdx", + "content/docs/releases/v17.mdx", + "content/docs/releases/v9.mdx", + "content/docs/ui/actions.mdx", + "content/docs/ui/apps.mdx", + "content/docs/ui/audience-based-interfaces.mdx", + "content/docs/ui/create-vs-edit-form.mdx", + "content/docs/ui/dashboards.mdx", + "content/docs/ui/doc-pages.mdx", + "content/docs/ui/field-grouping-and-order.mdx", + "content/docs/ui/forms.mdx", + "content/docs/ui/index.mdx", + "content/docs/ui/pages.mdx", + "content/docs/ui/public-data-collection.mdx", + "content/docs/ui/setup-app.mdx", + "content/docs/ui/translations.mdx", + "content/docs/ui/views.mdx", +] +// // Scope resolution. Omitting `args` entirely is the legitimate "audit // everything" invocation; supplying `args` but not a usable `args.docs` array @@ -43,6 +244,88 @@ if (args !== undefined && args !== null) { const DOCS = args && Array.isArray(args.docs) && args.docs.length ? args.docs : ALL_HANDWRITTEN log(`scope: ${DOCS.length} doc(s)${DOCS === ALL_HANDWRITTEN ? ' — FULL audit (no args.docs given)' : ''}`) +// --- Scope preflight: every path in scope must resolve to a real file --------- +// +// `pnpm check:docs-audit-scope` already keeps ALL_HANDWRITTEN honest in CI, but it +// can only see the DEFAULT list. A caller-supplied `args.docs` — the normal way this +// workflow is invoked — is checked by nothing, and a bad path there fails exactly the +// way #4851's stale list did: the audit agent finds no file, reports `fixCount: 0`, +// and the run summary shows a doc that was "audited clean". So resolve the scope +// first and refuse to start if anything in it is missing. +// +// This has to go through an agent: the workflow VM has no filesystem. That makes the +// preflight a REPORT rather than a measurement, so its arithmetic is checked below — +// a verdict that does not account for every path, exactly once, is treated as a +// failed preflight, not as a pass. A guard whose own answer is unverified is how +// #4868 happened. +const PREFLIGHT_SCHEMA = { + type: 'object', additionalProperties: false, + required: ['command', 'present', 'missing'], + properties: { + command: { type: 'string' }, + present: { type: 'array', items: { type: 'string' } }, + missing: { type: 'array', items: { type: 'string' } }, + }, +} + +phase('Scope Preflight') +const preflight = await agent( + `Resolve documentation paths against the repository working tree. This is a mechanical +existence check — do NOT read, summarise, judge or edit any file. + +PROCEDURE: +1. From the repository root, run ONE shell command that tests every path below, e.g. + \`for f in ; do [ -f "$f" ] || echo "MISSING $f"; done\` + (or an equivalent \`ls\`/\`test\` loop). Record the exact command you ran. +2. Return every path that resolves to an existing FILE under 'present', and every path + that does not under 'missing'. +3. Every path below must appear in exactly one of the two arrays, spelled EXACTLY as + given. Do not add, drop, normalise, deduplicate or re-order paths, and do not guess + at a corrected path for a missing one — reporting the miss IS the deliverable. + +PATHS (${DOCS.length}): +${DOCS.join('\n')}`, + { label: `preflight:${DOCS.length} path(s)`, phase: 'Scope Preflight', schema: PREFLIGHT_SCHEMA }, +) + +if (!preflight || !Array.isArray(preflight.present) || !Array.isArray(preflight.missing)) { + throw new Error( + '[docs-accuracy-audit] scope preflight returned no usable verdict. Refusing to audit ' + + DOCS.length + ' doc(s) whose paths were never resolved — an unresolved path produces ' + + 'an agent that reads nothing and reports "0 fixes", which is indistinguishable from a ' + + 'clean doc. Re-run the workflow.', + ) +} +{ + const accounted = [...preflight.present, ...preflight.missing] + const inScope = new Set(DOCS) + const seen = new Set(accounted) + const foreign = accounted.filter((p) => !inScope.has(p)) + const unaccounted = DOCS.filter((p) => !seen.has(p)) + if (foreign.length || unaccounted.length || accounted.length !== DOCS.length) { + throw new Error( + '[docs-accuracy-audit] scope preflight did not account for the scope exactly once: ' + + `${accounted.length} path(s) reported for ${DOCS.length} in scope` + + (unaccounted.length ? `; never reported: ${unaccounted.slice(0, 10).join(', ')}` : '') + + (foreign.length ? `; not in scope: ${foreign.slice(0, 10).join(', ')}` : '') + + '. A preflight that cannot be reconciled with its own input is a failed preflight — ' + + 'not a pass. Re-run the workflow.', + ) + } +} +if (preflight.missing.length) { + throw new Error( + `[docs-accuracy-audit] ${preflight.missing.length} of ${DOCS.length} doc path(s) in scope ` + + 'do not exist:\n ' + preflight.missing.join('\n ') + + '\n\nRefusing to run: an audit agent pointed at a non-existent file reads nothing and ' + + 'reports "0 fixes", so the run would report success over docs nobody looked at (#4851 — ' + + 'that is how #4781 and #4817 survived ~2 months of green full audits). If these came from ' + + 'args.docs, fix the caller; if they came from the default list, run ' + + '`node scripts/docs-audit/check-audit-scope.mjs --write`.', + ) +} +log(`preflight: all ${preflight.present.length} path(s) resolve (via \`${preflight.command}\`)`) + const PACKAGE_MAP = `ObjectStack is a metadata-driven application framework. Implementation lives in packages/: - packages/spec — Zod schemas for every metadata type (.zod.ts); source of truth for shapes & enums. Also packages/spec/src/{data,ui,...}. - packages/core — kernel: plugin system, service registry, lifecycle, events. @@ -82,11 +365,16 @@ const RULES = `HARD RULES: 6. Make minimal, precise edits — fix what is wrong, leave correct prose alone. 7. Verify code samples, CLI commands, API method names, config keys, env vars, file paths, enum values, and links against the actual implementation.` +// `docExists` is required and reported by the agent that actually opens the file — +// the preflight above is a separate call path, and #4868's lesson is that a self-check +// running somewhere other than the real path proves nothing about the real path. It is +// the difference between "audited, no problems found" and "there was nothing there". const FIX_LOG_SCHEMA = { type: 'object', additionalProperties: false, - required: ['doc', 'implementationFound', 'fixesApplied', 'fixCount', 'unresolved', 'notes'], + required: ['doc', 'docExists', 'implementationFound', 'fixesApplied', 'fixCount', 'unresolved', 'notes'], properties: { doc: { type: 'string' }, + docExists: { type: 'boolean' }, implementationFound: { type: 'boolean' }, fixesApplied: { type: 'array', items: { type: 'object', additionalProperties: false, required: ['category', 'summary', 'before', 'after', 'evidence'], @@ -126,7 +414,10 @@ ${HOUSE_FACTS} ${RULES} PROCEDURE: -1. Read the entire doc (${doc}). +1. Read the entire doc (${doc}). If that path does not exist, STOP: return docExists + false, fixCount 0, and say so in notes. Do NOT substitute a similar path, and do not + report "no inaccuracies" — a file you could not open was not audited, and the two + must never be reported the same way. 2. For each technical claim — code sample, CLI command, client/server API call, method/type name, config key, enum value, env var, file path, route, link — LOCATE the backing implementation under packages/ (Grep/Glob/Read; ripgrep via Bash is fine) and confirm whether the doc matches reality. 3. Apply evidence-backed fixes directly with Edit. Preserve frontmatter and MDX validity. 4. If a section documents a non-existent/removed/aspirational feature, remove it or qualify it as not-yet-implemented (with grep-empty evidence). @@ -178,6 +469,22 @@ const totalFixes = clean.reduce((n, r) => n + (r.fixLog?.fixCount || 0), 0) const totalRepairs = clean.reduce((n, r) => n + (r.verdict?.correctionsMade?.length || 0), 0) const totalResidual = clean.reduce((n, r) => n + (r.verdict?.residualInaccuracies?.length || 0), 0) +// The preflight said every path resolved; the agents that actually opened the files +// are the authority on whether that was true. If they disagree, the run did NOT audit +// what it claims to have audited — say so by failing, after logging the work that did +// land (the edits are already on disk) rather than returning a summary that reads green. +const ghosts = clean.filter((r) => r.fixLog?.docExists === false).map((r) => r.doc) +if (ghosts.length) { + log(`audited ${clean.length - ghosts.length} doc(s), ${totalFixes} fix(es), ${totalRepairs} verifier repair(s) before failing`) + throw new Error( + `[docs-accuracy-audit] ${ghosts.length} doc(s) in scope could not be opened by their audit ` + + 'agent, after the scope preflight reported every path as resolving:\n ' + ghosts.join('\n ') + + '\n\nThe preflight and the real read path disagree — trust the read path. Re-check the ' + + 'scope (`node scripts/docs-audit/check-audit-scope.mjs`) before believing any result ' + + 'from this run.', + ) +} + return { docsProcessed: clean.length, docsDropped: DOCS.length - clean.length, @@ -189,6 +496,7 @@ return { perDoc: clean.map((r) => ({ doc: r.doc, fixes: r.fixLog?.fixCount || 0, + docExists: r.fixLog?.docExists, implFound: r.fixLog?.implementationFound, confirmed: r.verdict?.confirmed, repairs: r.verdict?.correctionsMade?.length || 0, diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dcd8419ad0..a4de5a0440 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -94,6 +94,20 @@ jobs: - name: Doc/skill authoring guard run: pnpm check:doc-authoring + # #4851: the docs-accuracy-audit workflow carries its default scope inline + # (a workflow script runs in a vm with no filesystem, so it cannot enumerate + # content/docs/ itself). Hand-kept, that list rotted in BOTH directions — + # 16 entries pointing at files that no longer existed after the + # protocol/objectos → protocol/kernel rename, and 48 existing docs missing + # from it — while every "FULL audit" run reported green, which is how the + # accuracy defects in #4781 and #4817 survived ~2 months. This regenerates + # the list from the filesystem and fails when the two disagree either way. + # It lives in this job deliberately: the change that breaks the list is a + # docs rename, so a `packages/**` paths filter would blind the gate to + # exactly its own failure mode. + - name: Docs-audit scope is derived, not hand-kept + run: pnpm check:docs-audit-scope + # ADR-0090 D3 vocabulary ratchet: "role" is reserved-forbidden in docs # and skills. Existing occurrences are frozen in the baseline (better-auth # boundary, ARIA samples, educational mentions); NEW occurrences fail. diff --git a/package.json b/package.json index 6831dcbf82..abffa59fe0 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "check:i18n-coverage": "node scripts/check-i18n-coverage.mjs", "check:nul-bytes": "node scripts/check-nul-bytes.mjs", "check:doc-authoring": "node scripts/check-doc-authoring.mjs", + "check:docs-audit-scope": "node scripts/docs-audit/affected-docs.mjs --self-test && node scripts/docs-audit/check-audit-scope.mjs --self-test && node scripts/docs-audit/check-audit-scope.mjs", "check:role-word": "node scripts/check-role-word.mjs", "check:adr-anchors": "node scripts/check-adr-anchors.mjs", "check:org-identifier": "node scripts/check-org-identifier.mjs", diff --git a/scripts/docs-audit/README.md b/scripts/docs-audit/README.md index b4236e11d1..01542bd19d 100644 --- a/scripts/docs-audit/README.md +++ b/scripts/docs-audit/README.md @@ -81,6 +81,44 @@ looks like the obvious next step and is a provable no-op, for two independent re A hand-edited CHANGELOG outside a release is also close to nonexistent in practice. Left counted, and recorded here so the idea is not rediscovered as a gap. +## 1b. `check-audit-scope.mjs` — the audit workflow's scope, derived not hand-kept + +```bash +node scripts/docs-audit/check-audit-scope.mjs # verify (also: pnpm check:docs-audit-scope) +node scripts/docs-audit/check-audit-scope.mjs --write # regenerate the list from the filesystem +node scripts/docs-audit/check-audit-scope.mjs --self-test +``` + +The `docs-accuracy-audit` workflow (part 3) carries its default scope **inline**, as +`ALL_HANDWRITTEN`. It has to: a workflow script runs inside a `node:vm` context whose +only globals are `log`/`phase`/`console`/`budget`/timers plus +`agent`/`parallel`/`pipeline`/`workflow`/`args`, with code generation disabled — no +`require`, no `import`, no filesystem. It can neither walk `content/docs/` nor read a +JSON artifact, so the list cannot be derived *at run time*. + +It is therefore derived at *generation* time instead: `--write` rewrites the block from +`affected-docs.mjs --all` (one definition of "hand-written doc", not two), and the plain +run is a CI gate in `lint.yml` that fails when the block and `content/docs/` disagree +**in either direction**. + +Both directions matter, and only one had ever been noticed (#4851): + +- **listed but missing** — the 10 `content/docs/protocol/objectos/**` paths left behind + by the rename to `protocol/kernel/`, plus 6 others. An audit agent pointed at a + non-existent file reads nothing and reports `fixCount: 0`, which in the run summary is + indistinguishable from a doc that was checked and found accurate. That is how the + accuracy defects in #4781 and #4817 sat in `protocol/kernel/` for ~2 months while full + audits reported green. +- **exists but unlisted** — 48 docs, including all of `protocol/kernel/**` and the whole + `capabilities/` directory. A run logging `FULL audit (no args.docs given)` was + auditing 130 of 178 docs. + +The workflow additionally preflights its resolved scope — including a caller-supplied +`args.docs`, which no CI gate can see — and aborts naming any path that does not exist; +and each audit agent reports `docExists` from the read path itself, so a preflight that +was wrong cannot be laundered into a green summary. The gate covers the default list, +the preflight covers the caller's list, and the read path checks both. + ## 2. CI gate — `.github/workflows/docs-drift-check.yml` On any PR that touches `packages/**`, runs `affected-docs.mjs` against the base branch @@ -119,6 +157,8 @@ opens a PR when there are fixes. See the routine prompt for the exact steps. --- -**Cost note:** a full audit of all 128 hand-written docs is ~2.8M output tokens / ~160 -agents. Always prefer the change-scoped list (`affected-docs.mjs`) over `--all` except for -the periodic full backstop. +**Cost note:** a full audit is ~2 agents per doc — measured at ~2.8M output tokens / +~160 agents when the scope was 128 docs, and the hand-written set is 178 today (run +`check-audit-scope.mjs` for the current number; don't trust a count written down here). +Always prefer the change-scoped list (`affected-docs.mjs`) over `--all` except for the +periodic full backstop. diff --git a/scripts/docs-audit/affected-docs.mjs b/scripts/docs-audit/affected-docs.mjs index 442fd9c79c..abdff4f6e9 100644 --- a/scripts/docs-audit/affected-docs.mjs +++ b/scripts/docs-audit/affected-docs.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // Map a set of `packages/**` code changes to the hand-written docs that reference // the affected packages, so a doc-accuracy audit can be scoped to what actually -// changed instead of re-auditing all 128 hand-written docs every time. +// changed instead of re-auditing every hand-written doc (178 of them today) each time. // // Usage: // node scripts/docs-audit/affected-docs.mjs [sinceRef] # docs affected by changes since (default origin/main) diff --git a/scripts/docs-audit/check-audit-scope.mjs b/scripts/docs-audit/check-audit-scope.mjs new file mode 100644 index 0000000000..4b2991bc6d --- /dev/null +++ b/scripts/docs-audit/check-audit-scope.mjs @@ -0,0 +1,349 @@ +#!/usr/bin/env node +// Keep the `docs-accuracy-audit` workflow's default scope list DERIVED from the +// filesystem instead of hand-kept — and fail loudly the moment the two disagree. +// +// Usage: +// node scripts/docs-audit/check-audit-scope.mjs # verify; exit 1 naming every drifted entry +// node scripts/docs-audit/check-audit-scope.mjs --write # regenerate the block in place +// node scripts/docs-audit/check-audit-scope.mjs --self-test # pin the parser/renderer/differ (no repo state needed) +// +// ## Why this exists (#4851) +// +// `.claude/workflows/docs-accuracy-audit.js` carries the full list of hand-written +// docs inline, as `ALL_HANDWRITTEN`. It has to: a workflow script runs inside a +// `node:vm` context whose only globals are `log/phase/console/budget/setTimeout/ +// clearTimeout` plus `agent/parallel/pipeline/workflow/args`, with `codeGeneration` +// disabled — no `require`, no `import`, no filesystem, no `eval`. The script +// therefore cannot enumerate `content/docs/**` itself, and cannot read the list +// from a JSON artifact either. +// +// So the list was hand-kept, with a comment asking the next author to "keep in sync +// with `affected-docs.mjs --all`" — a promise nothing checked. It rotted, in BOTH +// directions, and neither direction announced itself: +// +// - 16 listed paths pointed at files that no longer exist (10 of them the whole +// `content/docs/protocol/objectos/**` directory, renamed to `protocol/kernel/`). +// A doc path that resolves to nothing produced an audit agent that read nothing +// and reported `fixCount: 0` — indistinguishable, in the run summary, from a doc +// that was audited and found accurate. #4781 and #4817 were both real accuracy +// defects in `protocol/kernel/`, both sat for ~2 months, and both were "covered" +// by green full-audit runs the entire time. +// - 48 docs on disk were absent from the list entirely — including all 9 of +// `protocol/kernel/**` and the whole `content/docs/capabilities/**` directory. +// A "FULL audit (no args.docs given)" run therefore audited 130 of 178 +// hand-written docs while calling itself full. +// +// The second direction is the larger hole and nobody had asked about it, which is +// the point: a hand-kept list drifts silently both ways. This gate closes both. +// It is the same discipline as #4690 / #4777 / #4804 / #4835 / #4868 / #4890 — a +// check whose subject has gone missing must go red, never green-by-vacancy. +// +// ## What "derived" means here +// +// The scope is not a curation: it is exactly `content/docs/**/*.mdx` minus +// `content/docs/references/**` (generated from packages/spec, audited by +// regenerating them). That definition already exists — `affected-docs.mjs --all` +// computes it for the change-scoped path — so this gate SHELLS OUT to that script +// rather than re-deriving it. One definition of "hand-written doc", one place to +// change it; a second walk here would be the next thing to drift out of sync. +// +// `--write` regenerates the inline block from that derivation, so the array is a +// generated artifact that happens to live inside a hand-written file. Hand-editing +// it is never necessary and this check will reject it. + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = execFileSync('git', ['rev-parse', '--show-toplevel'], { cwd: HERE }) + .toString() + .trim(); + +const WORKFLOW_REL = '.claude/workflows/docs-accuracy-audit.js'; +const BEGIN = '// '; +const END = '// '; + +const args = process.argv.slice(2); + +if (args.includes('--self-test')) { + selfTest(); + process.exit(0); +} + +// --- block extraction / rendering ------------------------------------------- + +/** + * The generated block's body (everything strictly between the markers), or throw. + * + * A missing marker is a HARD failure, not a skip. This gate's whole subject is a + * list that quietly stopped matching reality; a version of it that shrugs when it + * cannot find that list would reproduce the defect one level up (#4690's shape: + * no manifest, exit 0). + */ +export function extractBlock(source) { + const begin = source.indexOf(BEGIN); + const end = source.indexOf(END); + if (begin === -1 || end === -1 || end < begin) { + throw new Error( + `${WORKFLOW_REL}: could not find the generated scope block. Expected a region ` + + `delimited by "${BEGIN}" and "${END}". The audit workflow's default scope is ` + + `generated by this script; if the block was renamed or removed, restore it (or ` + + `update the markers here) — do NOT hand-maintain the list.`, + ); + } + return source.slice(begin + BEGIN.length, end); +} + +/** The doc paths declared inside the generated block. */ +export function parseBlock(source) { + const body = extractBlock(source); + // No doc path contains `]`, so "up to the first bracket" is unambiguous — and it + // parses the legacy single-line form too, so an old block is reported as drift + // rather than as a parse crash. + const m = body.match(/const ALL_HANDWRITTEN = (\[[^\]]*])/); + if (!m) { + throw new Error( + `${WORKFLOW_REL}: the generated scope block contains no \`const ALL_HANDWRITTEN = [...]\` ` + + `array literal. Regenerate it with \`node scripts/docs-audit/check-audit-scope.mjs --write\`.`, + ); + } + // The rendered form keeps a trailing comma on the last entry (smaller diffs when a + // doc is added); that is the one thing between it and plain JSON, so drop it. + const json = m[1].replace(/,(\s*])$/, '$1'); + let parsed; + try { + parsed = JSON.parse(json); + } catch (e) { + throw new Error( + `${WORKFLOW_REL}: ALL_HANDWRITTEN is not a plain array of string literals (${e.message}). ` + + `It is generated output — regenerate with \`--write\` rather than hand-editing it.`, + ); + } + return parsed; +} + +/** + * The exact text the block must contain for a given doc set. Byte-comparing against + * this is what makes hand-edits — including a merely reordered list — visible. + */ +export function renderBlock(docs) { + const entries = [...docs].sort(); + return [ + '', + '// GENERATED — do not hand-edit. `node scripts/docs-audit/check-audit-scope.mjs --write`', + '// derives this from the filesystem (every content/docs/**/*.mdx except', + '// references/, via `affected-docs.mjs --all`); the same script run without', + '// --write fails CI when the two disagree in either direction. See #4851: this', + '// list was hand-kept, 16 entries pointed at files that no longer existed and 48', + '// existing docs were absent from it, and a "FULL audit" reported green over both.', + 'const ALL_HANDWRITTEN = [', + ...entries.map((d) => ` ${JSON.stringify(d)},`), + ']', + '', + ].join('\n'); +} + +/** Splice a freshly rendered block back into the workflow source. */ +export function replaceBlock(source, docs) { + const begin = source.indexOf(BEGIN); + const end = source.indexOf(END); + if (begin === -1 || end === -1 || end < begin) throw new Error('missing markers'); // extractBlock already explains + return source.slice(0, begin + BEGIN.length) + renderBlock(docs) + source.slice(end); +} + +// --- the comparison ---------------------------------------------------------- + +/** + * Both directions, always. `dead` is what #4851 was filed about; `unlisted` is the + * direction nobody asked about and where 3x more drift had accumulated. Reporting + * only one of them would leave a gate that goes green while half-blind. + */ +export function diffScope(listed, derived) { + const listedSet = new Set(listed); + const derivedSet = new Set(derived); + const seen = new Set(); + const duplicates = []; + for (const d of listed) { + if (seen.has(d)) duplicates.push(d); + seen.add(d); + } + return { + dead: listed.filter((d) => !derivedSet.has(d)), + unlisted: derived.filter((d) => !listedSet.has(d)), + duplicates, + }; +} + +/** The hand-written doc set, from the ONE script that defines it. */ +function deriveDocs() { + const out = execFileSync( + process.execPath, + [join(REPO_ROOT, 'scripts/docs-audit/affected-docs.mjs'), '--all', '--json'], + { cwd: REPO_ROOT, stdio: ['ignore', 'pipe', 'ignore'] }, + ).toString(); + const docs = JSON.parse(out).docs; + if (!Array.isArray(docs) || docs.length === 0) { + throw new Error( + 'affected-docs.mjs --all --json returned no docs. Refusing to rewrite the audit ' + + 'scope to an empty list — an audit of nothing must not be able to report success.', + ); + } + return docs; +} + +// --- main -------------------------------------------------------------------- + +try { + main(); +} catch (e) { + // A structural failure (markers gone, list unparseable, derivation empty) is a + // RED result with a readable reason — never a stack trace, and never a pass. + console.error(`✗ ${e.message}`); + process.exit(1); +} + +function main() { + const workflowPath = join(REPO_ROOT, WORKFLOW_REL); + const source = readFileSync(workflowPath, 'utf8'); + const derived = deriveDocs(); + + if (args.includes('--write')) { + const before = parseBlock(source); + writeFileSync(workflowPath, replaceBlock(source, derived)); + const { dead, unlisted } = diffScope(before, derived); + console.log( + `✓ regenerated ${WORKFLOW_REL} scope: ${derived.length} hand-written doc(s) ` + + `(+${unlisted.length} added, -${dead.length} removed).`, + ); + return; + } + + const listed = parseBlock(source); + const { dead, unlisted, duplicates } = diffScope(listed, derived); + // Byte-compare too: correct paths in a hand-edited shape still means the block + // stopped being generated output, and the next `--write` would churn. + const blockDrift = extractBlock(source) !== renderBlock(derived); + + if (!dead.length && !unlisted.length && !duplicates.length && !blockDrift) { + console.log( + `✓ docs-accuracy-audit scope is in sync with content/docs/: ${listed.length} hand-written doc(s).`, + ); + return; + } + + console.error(`✗ ${WORKFLOW_REL}: ALL_HANDWRITTEN has drifted from content/docs/.\n`); + if (dead.length) { + console.error( + ` ${dead.length} listed path(s) do not exist — an audit agent pointed at one reads\n` + + ` nothing and reports "0 fixes", which in the run summary is indistinguishable\n` + + ` from a doc that was checked and found accurate:`, + ); + for (const d of dead) console.error(` - ${d}`); + console.error(''); + } + if (unlisted.length) { + console.error( + ` ${unlisted.length} hand-written doc(s) exist but are not listed — a run that calls\n` + + ` itself a "FULL audit" silently skips every one of them:`, + ); + for (const d of unlisted) console.error(` + ${d}`); + console.error(''); + } + if (duplicates.length) { + console.error(` ${duplicates.length} duplicate entr(ies): ${duplicates.join(', ')}\n`); + } + if (!dead.length && !unlisted.length && !duplicates.length && blockDrift) { + console.error( + ' The listed paths are correct but the block does not match its rendered form —\n' + + ' ordering, formatting or the header comment was hand-edited.\n', + ); + } + console.error(' Fix: node scripts/docs-audit/check-audit-scope.mjs --write'); + process.exit(1); +} + +// --- self-test --------------------------------------------------------------- + +/** + * Pin the three things that can silently break this gate: the block parser, the + * render round-trip, and the differ's SECOND direction. All hermetic — fixtures, + * no repo state — so a regression here fails on its own PR rather than being + * discovered the next time a directory is renamed. + */ +function selfTest() { + let failed = 0; + let total = 0; + const check = (label, want, got) => { + total++; + if (JSON.stringify(got) !== JSON.stringify(want)) { + console.error(` ✗ ${label}: expected ${JSON.stringify(want)}, got ${JSON.stringify(got)}`); + failed++; + } + }; + const throws = (label, fn, needle) => { + total++; + try { + fn(); + console.error(` ✗ ${label}: expected a throw, got none`); + failed++; + } catch (e) { + if (!String(e.message).includes(needle)) { + console.error(` ✗ ${label}: throw did not mention "${needle}" — got: ${e.message}`); + failed++; + } + } + }; + + const docs = ['content/docs/a.mdx', 'content/docs/b.mdx']; + const file = `header\n${BEGIN}${renderBlock(docs)}${END}\nfooter\n`; + + check('parseBlock round-trips the rendered block', docs, parseBlock(file)); + check('renderBlock sorts', ['a', 'b'], parseBlock(`${BEGIN}${renderBlock(['b', 'a'])}${END}`)); + check( + 'replaceBlock preserves the surrounding file', + ['header', 'footer', ''], + (() => { + const out = replaceBlock(file, ['content/docs/c.mdx']); + return [out.split('\n')[0], out.split('\n').at(-2), out.split('\n').at(-1)]; + })(), + ); + check('replaceBlock installs the new list', ['content/docs/c.mdx'], parseBlock(replaceBlock(file, ['content/docs/c.mdx']))); + + // A gate that cannot find its subject must fail, not pass. This is the exact + // failure mode the gate exists to prevent, applied to the gate itself. + throws('missing markers throw', () => extractBlock('no markers here'), 'could not find the generated scope block'); + throws( + 'a block with no array literal throws', + () => parseBlock(`${BEGIN}\n// nothing here\n${END}`), + 'no `const ALL_HANDWRITTEN = [...]` array literal', + ); + throws( + 'a hand-edited (non-JSON) array literal throws', + () => parseBlock(`${BEGIN}\nconst ALL_HANDWRITTEN = [\n 'a.mdx',\n]\n${END}`), + 'not a plain array of string literals', + ); + + // Both directions of drift, each pinned on its own — #4851 asked only about + // `dead`, and `unlisted` was where 3x more of the rot actually was. + const d1 = diffScope(['x.mdx', 'gone.mdx'], ['x.mdx', 'new.mdx']); + check('dead entries are reported', ['gone.mdx'], d1.dead); + check('unlisted docs are reported', ['new.mdx'], d1.unlisted); + check('no false positives when in sync', { dead: [], unlisted: [], duplicates: [] }, diffScope(['x.mdx'], ['x.mdx'])); + check('duplicates are reported', ['x.mdx'], diffScope(['x.mdx', 'x.mdx'], ['x.mdx']).duplicates); + + // The renamed-directory case that opened #4851, end to end through the differ. + const renamed = diffScope( + ['content/docs/protocol/objectos/index.mdx'], + ['content/docs/protocol/kernel/index.mdx'], + ); + check('a renamed directory shows up on BOTH sides', 1, renamed.dead.length); + check('…and its new home is flagged as unlisted', 1, renamed.unlisted.length); + + if (failed) { + console.error(`\n✗ check-audit-scope self-test failed (${failed} case(s)).`); + process.exit(1); + } + console.log(`✓ check-audit-scope self-test: ${total} cases pass.`); +}