From 994322dc69cf41566792c8de752de2c8234ce577 Mon Sep 17 00:00:00 2001 From: lex Date: Sun, 9 Aug 2026 11:49:33 +0800 Subject: [PATCH] test(dag): enforce core behavior coverage --- .github/workflows/ci-typecheck.yml | 16 +- docs/dag-core-tdd-ci-matrix.md | 43 +++++ packages/opencode/package.json | 1 + packages/opencode/script/dag-core-coverage.ts | 166 ++++++++++++++++++ .../test/dag/dag-core-coverage-gate.test.ts | 19 ++ .../opencode/test/dag/workflow-tool.test.ts | 79 +++++++++ packages/schema/test/event-manifest.test.ts | 12 +- 7 files changed, 329 insertions(+), 7 deletions(-) create mode 100644 docs/dag-core-tdd-ci-matrix.md create mode 100644 packages/opencode/script/dag-core-coverage.ts create mode 100644 packages/opencode/test/dag/dag-core-coverage-gate.test.ts diff --git a/.github/workflows/ci-typecheck.yml b/.github/workflows/ci-typecheck.yml index dbcc1bb3fd..18ec07b500 100644 --- a/.github/workflows/ci-typecheck.yml +++ b/.github/workflows/ci-typecheck.yml @@ -1,14 +1,13 @@ # ============================================================================ # ๐Ÿ” CI ยท Typecheck # ---------------------------------------------------------------------------- -# Purpose : TypeScript type checking across all packages (bun typecheck) -# plus oxlint warning ratchet (bun run lint, --max-warnings gate) +# Purpose : TypeScript type checking across all packages (bun typecheck), +# oxlint warning ratchet, and the DAG core behavior/coverage gate # Trigger : Push to `main`/`dev`, PRs targeting `main`/`dev`, manual dispatch # Jobs : typecheck โ€” single Linux runner, `bun run lint` + `bun typecheck` -# Gate : Required status check on BOTH `dev` and `main` rulesets โ€” it is -# the fast gate for feat/fix โ†’ dev PRs (full test suite only gates -# dev โ†’ main, see ci-test.yml). Lint lives inside this job so it -# blocks merges without editing the rulesets' required checks. +# Gate : Required status check on BOTH `dev` and `main` rulesets. The DAG +# core gate protects state-machine and persistence changes before +# they merge to dev; the full suite still gates dev โ†’ main. # Notes : No push trigger on feat/* or fix/* (frequent changes); PRs cover # them. # ============================================================================ @@ -42,3 +41,8 @@ jobs: - name: Run typecheck run: bun typecheck + + - name: Run DAG core behavior and coverage gate + working-directory: packages/opencode + run: bun run test:dag-core + timeout-minutes: 10 diff --git a/docs/dag-core-tdd-ci-matrix.md b/docs/dag-core-tdd-ci-matrix.md new file mode 100644 index 0000000000..6b066f9f02 --- /dev/null +++ b/docs/dag-core-tdd-ci-matrix.md @@ -0,0 +1,43 @@ +# DAG core TDD / CI coverage matrix + +Baseline: `origin/dev` at `d482e4bb6` (2026-08-09). This matrix treats public behavior as the unit of coverage. It does not use private-helper tests or a repository-wide percentage as a proxy for correctness. + +## Confirmed seams + +| Seam | Core invariant | Existing behavior tests | Current CI enforcement | Finding | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Workflow tool / `Dag.Service` | Start, extend, replan and controls publish only legal durable transitions; invalid input has no side effects | `workflow-tool.test.ts`, `dag-create-validation.test.ts`, `dag-step-semantics.test.ts`, `dag-node-started-guard.test.ts`, `dag-dynamic-correctness.test.ts` | New DAG gate runs on PRs to `dev` and `main` | **Closed here:** one public-seam test now drives pause/resume/cancel/complete/step through `WorkflowTool.execute` and checks the durable event type. | +| Durable store โ†’ replay โ†’ recovery | An acknowledged wake is atomic; replay rebuilds the same read model; a restart neither duplicates work nor leaves invented running ownership | `dag-store-wake.test.ts`, `dag-replay-idempotency.test.ts`, `dag-deadline-extended.test.ts`, `dag-recovery.test.ts`, `dag-loop-recovery-integration.test.ts`, `dag-orphan-pending-recovery.test.ts` | New DAG gate runs the behavior suites and enforces critical store/projector floors | Behavior is strong; the missing PR gate and coverage floor are closed here. Store-defect retry semantics remain intentionally undecided, so they are not a missing test for an accepted behavior yet. | +| Runtime state machine | Wake batches are stable and retryable; spawn owns at most one attempt; abort settles live children; lock timeout releases; escalation is delivered before adjudication; every terminal path clears pending escalation | `dag-wake-integration.test.ts`, `spawn-completion.test.ts`, `dag-loop-guards.test.ts`, `dag-workflow-lock.test.ts`, `dag-timeout-escalation*.test.ts`, `dag-escalation-clear-flag.test.ts`, race regressions | New DAG gate runs the complete OpenCode `test/dag` directory | Core P0/P1 behavior is covered. The prior gap was enforcement and measured floors, not another broad behavior suite. | +| Durable event / SDK / TUI projection | Event folds are replay-safe; generated SDK stays fresh; summary events replace the server-derived view and recover missed SSE updates by refetching | `dag-projector-drift.test.ts`, `dag-summary-publisher*.test.ts`, `sync-dag.test.tsx`, `dag-inspector*.test.ts`; SDK `check:generated` | New DAG gate runs Schema, SDK freshness and TUI contract suites on both PR targets | **Closed here:** every DAG durable definition is now checked at its versioned manifest key. The TUI behavior itself was already covered. | +| CI quality gate | A DAG-critical regression must fail before merge to `dev`; critical public modules may not silently lose tested lines/functions | `bun turbo test` covers all package tests; Linux unit + Linux/Windows app E2E; typecheck/lint on `dev` and `main` PRs | `ci-typecheck.yml` now runs `bun run test:dag-core` on PRs to `dev` and `main` | **Closed here:** package-scoped behavior suites plus fixed per-module line/function floors. Bun 1.3.14 LCOV contains no branch records, so transition-matrix tests remain the branch-equivalent guard. | + +## Measured baseline + +| Surface | Tests | Relevant measured coverage | +| -------------------------------------------------- | ----: | --------------------------------------------------------------------------------------------------------------- | +| Core graph / scheduling / transition / store suite | 90 | scheduling 94.56% lines; transitions 96.00%; types 94.78%; store 78.34% in the Core-only slice | +| OpenCode DAG public/runtime suite | 397 | `dag.ts` 99.55%; loop 93.73%; recovery 99.30%; spawn 94.58%; projector 100%; store 87.43%; workflow tool 90.80% | +| TUI DAG projection / inspector suite | 50 | inspector utils 100%; inspector 91.69%; sync DAG reducer/bootstrap/reconnect has five named behavior tests | + +The Core-only projector number (19.23%) is not a defect: projector behavior lives primarily in the OpenCode integration slice, where the same public projector reaches 100% lines. Coverage must therefore be evaluated per agreed suite, not by averaging unrelated package imports. + +## Enforced gate policy + +`packages/opencode/script/dag-core-coverage.ts` now: + +1. run the Core, OpenCode, Schema and TUI DAG contract suites from their package directories; +2. run generated SDK freshness validation; +3. parse LCOV for explicitly named critical files; +4. fail below conservative line/function floors with enough headroom for harmless refactors; +5. keep fault-injection/retry policy out until its shutdown and backoff contract is designed. + +The floors are fixed in source and intentionally sit below the measured baseline: Core critical modules range from 70โ€“94% lines and 60โ€“95% functions; OpenCode critical modules range from 85โ€“98% lines and 70โ€“95% functions; TUI critical modules range from 90โ€“98% lines and 88โ€“95% functions. A future change may raise them, but lowering them requires an explicit code review diff. + +## Red โ†’ green evidence + +1. Coverage evaluator: `bun test test/dag/dag-core-coverage-gate.test.ts` first failed with `Cannot find module '../../script/dag-core-coverage'`; after the minimal parser/assertion implementation it passed `1 pass, 0 fail`. +2. Public control dispatch: with the cancel branch deliberately routed to pause, `bun test test/dag/workflow-tool.test.ts --test-name-pattern 'dispatches every public control operation'` failed with `Expected: "dag.workflow.cancelled"; Received: "dag.workflow.paused"`; after restoring the correct public behavior it passed `1 pass, 0 fail`. +3. Durable manifest membership: with the DAG inventory deliberately omitted, `bun test test/event-manifest.test.ts --test-name-pattern 'registers every DAG durable event'` failed with `Received: undefined`; after restoring the inventory it passed `1 pass, 20 expect() calls, 0 fail`. +4. CI failure proof: with only the loop line floor temporarily raised from 90% to 99%, `bun run test:dag-core` ran `399 pass, 0 fail` and still exited 1 with `loop.ts: lines 93.73% < 99.00%`. The committed floor is restored to 90%; this proves coverage loss fails the same command invoked by CI. +5. Final gate: `bun run test:dag-core` passed Core `90`, OpenCode `399`, Schema `3`, and TUI `50` tests; SDK regeneration produced no diff and every critical-file floor passed. diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 5d4d5f7d23..ccb6e33279 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -8,6 +8,7 @@ "scripts": { "typecheck": "tsgo --noEmit", "test": "bun test --timeout 30000 --only-failures", + "test:dag-core": "bun run script/dag-core-coverage.ts", "test:httpapi": "bun run script/httpapi-exercise.ts --mode coverage --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode auth --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode effect --fail-on-missing --fail-on-skip", "test:httpapi:ci": "bun run script/httpapi-exercise.ts --mode coverage --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode auth --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode effect --fail-on-missing --fail-on-skip --progress --trace", "bench:test": "bun run script/bench-test-suite.ts", diff --git a/packages/opencode/script/dag-core-coverage.ts b/packages/opencode/script/dag-core-coverage.ts new file mode 100644 index 0000000000..9eb0404a0f --- /dev/null +++ b/packages/opencode/script/dag-core-coverage.ts @@ -0,0 +1,166 @@ +import { mkdtemp, rm } from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +export type CoverageRecord = { + lines: { found: number; hit: number } + functions: { found: number; hit: number } +} + +export type CoverageThreshold = { + file: string + lines: number + functions: number +} + +export function parseLcov(input: string) { + return input + .split("end_of_record") + .map((record) => record.trim().split(/\r?\n/)) + .reduce((report, lines) => { + const file = field(lines, "SF") + if (!file) return report + report.set(file, { + lines: { + found: Number(field(lines, "LF") ?? 0), + hit: Number(field(lines, "LH") ?? 0), + }, + functions: { + found: Number(field(lines, "FNF") ?? 0), + hit: Number(field(lines, "FNH") ?? 0), + }, + }) + return report + }, new Map()) +} + +export function assertCoverage(report: ReadonlyMap, thresholds: readonly CoverageThreshold[]) { + const failures = thresholds.flatMap((threshold) => { + const record = report.get(threshold.file) + if (!record) return [`${threshold.file}: missing from LCOV report`] + const lines = percentage(record.lines) + const functions = percentage(record.functions) + return [ + ...(lines < threshold.lines + ? [`${threshold.file}: lines ${lines.toFixed(2)}% < ${threshold.lines.toFixed(2)}%`] + : []), + ...(functions < threshold.functions + ? [`${threshold.file}: functions ${functions.toFixed(2)}% < ${threshold.functions.toFixed(2)}%`] + : []), + ] + }) + if (failures.length > 0) throw new Error(`DAG core coverage gate failed:\n${failures.join("\n")}`) +} + +export async function runDagCoreCoverageGate() { + const root = path.resolve(import.meta.dir, "../../..") + const output = await mkdtemp(path.join(os.tmpdir(), "opencode-dag-core-coverage-")) + try { + yieldMessage("Core state machine, store, and transition seams") + await runCoverageSuite({ + cwd: path.join(root, "packages/core"), + output: path.join(output, "core"), + tests: [ + "test/dag-core.test.ts", + "test/dag-store-wake.test.ts", + "test/dag-node-cancelled-projection.test.ts", + "test/dag-store-summaries.test.ts", + "test/dag-projector-drift.test.ts", + ], + thresholds: [ + { file: "src/dag/core/graph.ts", lines: 70, functions: 60 }, + { file: "src/dag/core/replan.ts", lines: 90, functions: 95 }, + { file: "src/dag/core/scheduling.ts", lines: 92, functions: 80 }, + { file: "src/dag/core/transitions.ts", lines: 94, functions: 75 }, + { file: "src/dag/core/types.ts", lines: 92, functions: 85 }, + { file: "src/dag/store.ts", lines: 75, functions: 65 }, + ], + }) + + yieldMessage("OpenCode DAG public API and runtime seams") + await runCoverageSuite({ + cwd: path.join(root, "packages/opencode"), + output: path.join(output, "opencode"), + tests: ["test/dag"], + thresholds: [ + { file: "../core/src/dag/projector.ts", lines: 98, functions: 95 }, + { file: "../core/src/dag/store.ts", lines: 85, functions: 80 }, + { file: "src/dag/dag.ts", lines: 98, functions: 95 }, + { file: "src/dag/runtime/loop.ts", lines: 90, functions: 88 }, + { file: "src/dag/runtime/recovery.ts", lines: 95, functions: 75 }, + { file: "src/dag/runtime/spawn.ts", lines: 90, functions: 70 }, + { file: "src/dag/runtime/summary-publisher.ts", lines: 95, functions: 90 }, + { file: "src/tool/workflow.ts", lines: 88, functions: 75 }, + ], + }) + + yieldMessage("Schema manifest and generated SDK contract") + await run(["bun", "test", "test/event-manifest.test.ts", "--only-failures"], path.join(root, "packages/schema")) + await run(["bun", "run", "check:generated"], path.join(root, "packages/sdk/js")) + + yieldMessage("TUI projection and inspector seams") + await runCoverageSuite({ + cwd: path.join(root, "packages/tui"), + output: path.join(output, "tui"), + tests: [ + "test/cli/cmd/tui/sync-dag.test.tsx", + "test/feature-plugins/dag-inspector.test.tsx", + "test/feature-plugins/dag-inspector-utils.test.ts", + ], + thresholds: [ + { file: "src/feature-plugins/system/dag-inspector-utils.ts", lines: 98, functions: 95 }, + { file: "src/feature-plugins/system/dag-inspector.tsx", lines: 90, functions: 88 }, + ], + }) + } finally { + await rm(output, { recursive: true, force: true }) + } +} + +function field(lines: readonly string[], key: string) { + const prefix = `${key}:` + const line = lines.find((item) => item.startsWith(prefix)) + return line?.slice(prefix.length) +} + +function percentage(value: { found: number; hit: number }) { + if (value.found === 0) return 100 + return (value.hit / value.found) * 100 +} + +async function runCoverageSuite(input: { + cwd: string + output: string + tests: string[] + thresholds: CoverageThreshold[] +}) { + await run( + [ + "bun", + "test", + ...input.tests, + "--only-failures", + "--timeout=30000", + "--coverage", + "--coverage-reporter=lcov", + `--coverage-dir=${input.output}`, + ], + input.cwd, + ) + assertCoverage(parseLcov(await Bun.file(path.join(input.output, "lcov.info")).text()), input.thresholds) +} + +async function run(command: string[], cwd: string) { + const child = Bun.spawn(command, { cwd, stdout: "inherit", stderr: "inherit" }) + const exitCode = await child.exited + if (exitCode !== 0) throw new Error(`${command.join(" ")} failed with exit code ${exitCode}`) +} + +function yieldMessage(message: string) { + console.log(`\n[DAG core gate] ${message}`) +} + +if (import.meta.main) { + await runDagCoreCoverageGate() + console.log("\n[DAG core gate] all critical behavior and coverage floors passed") +} diff --git a/packages/opencode/test/dag/dag-core-coverage-gate.test.ts b/packages/opencode/test/dag/dag-core-coverage-gate.test.ts new file mode 100644 index 0000000000..c5afeaa2ae --- /dev/null +++ b/packages/opencode/test/dag/dag-core-coverage-gate.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "bun:test" +import { assertCoverage, parseLcov } from "../../script/dag-core-coverage" + +describe("DAG core coverage gate", () => { + it("rejects a critical public module below its line floor", () => { + const report = parseLcov(` +SF:src/dag/runtime/loop.ts +FNF:10 +FNH:9 +LF:100 +LH:89 +end_of_record +`) + + expect(() => assertCoverage(report, [{ file: "src/dag/runtime/loop.ts", lines: 90, functions: 80 }])).toThrow( + "src/dag/runtime/loop.ts: lines 89.00% < 90.00%", + ) + }) +}) diff --git a/packages/opencode/test/dag/workflow-tool.test.ts b/packages/opencode/test/dag/workflow-tool.test.ts index 610b2db967..4826306568 100644 --- a/packages/opencode/test/dag/workflow-tool.test.ts +++ b/packages/opencode/test/dag/workflow-tool.test.ts @@ -105,6 +105,21 @@ const store = Layer.mock(DagStore.Service, { timeCreated: 1, timeUpdated: 2, } + : id === "dag_paused" || id === "dag_step" + ? { + id, + projectId: projectID, + sessionId: "ses_workflow_parent", + title: "Control workflow", + status: id === "dag_paused" ? "paused" : "running", + config: "{}", + seq: 1, + wakeReported: false, + startedAt: 1, + completedAt: null, + timeCreated: 1, + timeUpdated: 2, + } : id === "dag_deep_status" ? { id, @@ -216,6 +231,34 @@ const store = Layer.mock(DagStore.Service, { timeCreated: 1, timeUpdated: 2, }] + : id === "dag_step" + ? [{ + id: "node_ready", + workflowId: "dag_step", + name: "Ready node", + workerType: "build", + status: "pending", + required: true, + dependsOn: [], + modelId: null, + modelProviderId: null, + childSessionId: null, + output: null, + capturedOutput: null, + errorReason: null, + errorClass: null, + deadlineMs: null, + wakeEligible: false, + wakeReported: false, + replanAttempts: 0, + seq: 1, + timeoutExtensions: 0, + escalationPending: false, + startedAt: null, + completedAt: null, + timeCreated: 1, + timeUpdated: 1, + }] : [], ), }) @@ -443,6 +486,42 @@ describe("workflow tool execution", () => { }), ) + runtime.effect("dispatches every public control operation to its durable workflow event", () => + Effect.gen(function* () { + const info = yield* WorkflowTool + const workflow = yield* info.init() + const controls = [ + { workflowID: "dag_status", operation: "pause" }, + { workflowID: "dag_paused", operation: "resume" }, + { workflowID: "dag_status", operation: "cancel" }, + { workflowID: "dag_status", operation: "complete" }, + { workflowID: "dag_step", operation: "step" }, + ] as const + const routed = yield* Effect.forEach(controls, (control) => + Effect.gen(function* () { + published.length = 0 + yield* workflow.execute( + { + action: "control", + workflow_id: control.workflowID, + operation: control.operation, + }, + toolContext(), + ) + return published.find((event) => event.type.startsWith("dag.workflow."))?.type ?? "missing" + }), + ) + + expect(routed).toEqual([ + DagEvent.WorkflowPaused.type, + DagEvent.WorkflowResumed.type, + DagEvent.WorkflowCancelled.type, + DagEvent.WorkflowCompleted.type, + DagEvent.WorkflowStepped.type, + ]) + }), + ) + runtime.effect("starts from an inline structured spec without a file", () => Effect.gen(function* () { published.length = 0 diff --git a/packages/schema/test/event-manifest.test.ts b/packages/schema/test/event-manifest.test.ts index e48bb49b0f..6141c23702 100644 --- a/packages/schema/test/event-manifest.test.ts +++ b/packages/schema/test/event-manifest.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" -import { FileSystem, Integration, Permission, Project, Reference, Session, Workspace } from "../src" +import { Event, FileSystem, Integration, Permission, Project, Reference, Session, Workspace } from "../src" +import { DagEvent } from "../src/dag-event" import { EventManifest } from "../src/event-manifest" import { IdeEvent } from "../src/ide-event" import { SessionEvent } from "../src/session-event" @@ -50,4 +51,13 @@ describe("public event manifest", () => { expect(EventManifest.Durable.has("session.next.step.ended.1")).toBe(false) expect(EventManifest.Durable.get("session.next.step.ended.2")).toBe(SessionEvent.Step.Ended) }) + + test("registers every DAG durable event under its versioned public key", () => { + DagEvent.DurableDefinitions.forEach((definition) => { + if (!definition.durable) throw new Error(`${definition.type} is missing durable metadata`) + expect(EventManifest.Durable.get(Event.versionedType(definition.type, definition.durable.version))).toBe( + definition, + ) + }) + }) })