From 9eb9538ec8ef1ce0783813298400ba40c82e9c8e Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 21:13:54 -0700 Subject: [PATCH 1/7] feat: add deterministic dbt fault injection (`altimate-code fault-injection`, `dbt_fault_injection`) Corrupts one upstream relation in a sandbox copy of a DuckDB database, rebuilds everything downstream, runs the project's own tests, and reports which faults the tests miss, which downstream models silently changed, and a proposed dbt test that would catch each. The engine is `FaultInjectionSession` in `@altimateai/altimate-core`; this change adds the caller. - driver and report formatter beside the data-diff ones (`fault-injection.ts`, `fault-injection-report.ts`) - agent tool `dbt_fault_injection`, registered only when the engine class is present in the installed core - `altimate-code fault-injection` command, no model or API key needed - works only on copies of the database and project; refuses in-memory and MotherDuck databases, profiles that attach other databases, and hooks that run `ATTACH`, `COPY` or `EXPORT`; cleans up on success, failure and interrupt - `temp_directory` from the user's profile is redirected into the work dir - `ALTIMATE_CORE_DEV_PATH` (development only, ignored on release channels) loads a locally built engine - `FAULT_INJECTION_MIN_CORE_VERSION` is a placeholder until the core release exists - tests with a self-contained fixture, and docs Co-Authored-By: Claude Opus 5.5 (1M context) --- CONTRIBUTING.md | 18 + docs/docs/data-engineering/tools/dbt-tools.md | 155 ++ docs/docs/data-engineering/tools/index.md | 2 +- docs/docs/usage/cli.md | 1 + .../connections/fault-injection-report.ts | 339 ++++ .../native/connections/fault-injection.ts | 1478 +++++++++++++++++ .../altimate/native/connections/register.ts | 8 + .../opencode/src/altimate/native/types.ts | 81 + .../src/altimate/tools/dbt-fault-injection.ts | 99 ++ .../opencode/src/cli/cmd/fault-injection.ts | 152 ++ packages/opencode/src/index.ts | 7 + packages/opencode/src/tool/registry.ts | 10 + .../carry-forward/tools-present.test.ts | 2 + .../fault-injection-dbt-verify.test.ts | 399 +++++ .../altimate/fault-injection-driver.test.ts | 1279 ++++++++++++++ .../altimate/fault-injection-duckdb.test.ts | 234 +++ .../test/altimate/fault-injection-e2e.test.ts | 279 ++++ .../fault-injection-registration.test.ts | 84 + .../fault-injection/project/README.md | 9 + .../fault-injection/project/dbt_project.yml | 18 + .../project/models/customer_orders.sql | 9 + .../fault-injection/project/models/schema.yml | 35 + .../project/models/stg_orders.sql | 9 + .../fault-injection/project/profiles.yml | 8 + .../project/seeds/raw_customers.csv | 7 + .../project/seeds/raw_orders.csv | 25 + 26 files changed, 4746 insertions(+), 1 deletion(-) create mode 100644 packages/opencode/src/altimate/native/connections/fault-injection-report.ts create mode 100644 packages/opencode/src/altimate/native/connections/fault-injection.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-fault-injection.ts create mode 100644 packages/opencode/src/cli/cmd/fault-injection.ts create mode 100644 packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts create mode 100644 packages/opencode/test/altimate/fault-injection-driver.test.ts create mode 100644 packages/opencode/test/altimate/fault-injection-duckdb.test.ts create mode 100644 packages/opencode/test/altimate/fault-injection-e2e.test.ts create mode 100644 packages/opencode/test/altimate/fault-injection-registration.test.ts create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/README.md create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/dbt_project.yml create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/models/customer_orders.sql create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/models/schema.yml create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/models/stg_orders.sql create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/profiles.yml create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_customers.csv create mode 100644 packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_orders.csv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d4fb78c67..f436756ace 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,24 @@ bun run local # build + run The script handles platform detection (including Rosetta 2), `NODE_PATH` setup for native modules like `@altimateai/altimate-core`, and binary resolution. +#### Running fault injection against a locally built altimate-core + +`altimate-code fault-injection` and the `dbt_fault_injection` tool need `FaultInjectionSession` +from `@altimateai/altimate-core`. To develop against a core build that is not published yet, point +`ALTIMATE_CORE_DEV_PATH` at the built Node binding (the `crates/altimate-core-node` directory of an +altimate-core checkout, after `npm run build` there). This is for development only: it loads a +native addon from an arbitrary path, so published releases (`latest`, `beta`) ignore it, and the +report says when it was used. + +```bash +ALTIMATE_CORE_DEV_PATH=~/code/altimate-core/crates/altimate-core-node \ +ALTIMATE_DBT_PATH=~/venvs/dbt-duckdb/bin/dbt \ + bun dev fault-injection path/to/dbt-project +``` + +The same two variables enable `test/altimate/fault-injection-e2e.test.ts`, which is skipped when +dbt-duckdb or the engine class is unavailable. + To compile all 12 platform targets (CI/release): ```bash diff --git a/docs/docs/data-engineering/tools/dbt-tools.md b/docs/docs/data-engineering/tools/dbt-tools.md index 79714e1bb5..09e2e0ba9c 100644 --- a/docs/docs/data-engineering/tools/dbt-tools.md +++ b/docs/docs/data-engineering/tools/dbt-tools.md @@ -140,6 +140,161 @@ unit_tests: --- +## dbt_fault_injection + +Find the upstream data faults a dbt project's own tests would miss. + +It corrupts one upstream relation at a time in a private copy of the database, rebuilds every model +downstream of it, runs the project's tests, and reports which faults no test noticed. It is +deterministic and uses no model, so it is also available as a plain command that needs no API key: + +```bash +altimate-code fault-injection # the dbt project in the current directory +altimate-code fault-injection path/to/project --budget 50 +altimate-code fault-injection --model stg_orders # corrupt only this model, seed or source +altimate-code fault-injection --format json --fail-under 60 # for CI +``` + +**Parameters** (tool) / **flags** (command): +- `project_dir` / `[project]` (optional): dbt project root. Defaults to the working directory +- `budget` / `--budget` (optional, default 20): maximum number of faults to inject +- `model` / `--model` (optional): corrupt only this model, seed, snapshot or source +- `target` / `--target`, `profiles_dir` / `--profiles-dir` (optional): as for dbt +- `--seed`, `--work-dir`, `--format text|json`, `--fail-under `: command only + +The seven faults are duplicated rows, dropped rows, a column set to NULL, a number multiplied by 100, +a category value never seen before, a date moved one day, and a foreign key pointing nowhere. Each +touches 5% of the eligible rows of one relation (at least one row). + +### Reading the output + +A real run on the 5-model jaffle_shop project, trimmed (`...` marks removed lines): + +``` +Fault injection: jaffle_shop_snowflake (duckdb) + +Catch rate: 68.0% (34 of 50 faults that mattered were caught) + 50 faults injected: 34 caught, 16 slipped through, 0 harmless, 0 invalid + ... + +Slipped through (16): the project's tests did not notice these + +1. seed raw_customers: `first_name` set to NULL in 5 of 100 rows + Fault id: jaffle_shop_snowflake|seed.jaffle_shop_snowflake.raw_customers|null_out|first_name + 5 tests ran and none failed because of the fault. Changed downstream: + - model customers: 5 rows changed (first_name: 5) of 100 (matched on customer_id) + - model sample: content changed, row count unchanged at 100; no unique key is declared for this relation, so no row-level detail + ... + Proposed test: not_null + Verified on the data: passes on the clean data and fails on the corrupted copy (5 failing). + seeds: + - name: raw_customers + columns: + - name: first_name + data_tests: + - not_null + Note: `first_name` has no NULL in the baseline, so a `not_null` test passes today and fails on the first NULL. +... +3. seed raw_orders: `order_date` moved one day later in 5 of 99 rows + ... + No test is proposed for this fault: `order_date` is a date or timestamp, and a one-day shift stays inside the span the data legitimately covers. ... +... +8. seed raw_payments: `amount` multiplied by 100 in 6 of 113 rows + ... + Proposed test: range (singular test) + Verified on the data: passes on the clean data and fails on the corrupted copy (1 failing). + The standard test for this needs a package this project has not installed, so this is a singular test (plain SQL, no package needed). + Save as tests/fault_injection/fault_injection__raw_payments__range__amount.sql (or in another folder listed under test-paths in dbt_project.yml): + SELECT * FROM {{ ref('raw_payments') }} WHERE "amount" < 0 OR "amount" > 100000 +... +These are gaps in the project's tests. They are not evidence that the data in the warehouse today is wrong. +Took 195s: setup and baseline build 6.1s; 50 faults 139s (2.8s each on average); profiling and no-fault controls 51s. +``` + +- **Caught**: a test failed, or a downstream model failed to build +- **Slipped through**: no test failed and downstream data changed. These are the findings +- **Harmless**: nothing downstream changed, so the fault is left out of the catch rate +- **Invalid**: the fault touched no row, or its sandbox failed twice +- **Catch rate**: caught / (caught + slipped through), rounded down + +Each slipped fault lists the downstream models that changed, row by row where the model has a +declared unique key and by row count and checksum otherwise, and either a test that would catch it +or the reason there is none. + +A proposed test is offered only if it passes on the clean data and fails on the corrupted copy (both +are checked by running it) and is unlikely to be a snapshot of today's data. So: + +- A range is not the column's current minimum and maximum. It leaves an order of magnitude of room, + and a fault that stays inside that room gets no range test. +- A column computed from the current date or time (found from the model's SQL, through its parents, + and from relations that change on every rebuild with no fault injected) gets no range, no list of + values and no non-null-share floor, because they would fail as the clock moves. +- A list of accepted values is proposed only for a column that is plausibly a category: at most 20 + distinct values, at least 30 non-null rows and at least 10 rows per value. Such a column is also the + only kind the "new category value" fault is injected into. +- A date moved by one day gets no test: any range narrow enough to see it fails when the next row arrives. +- A test for a model or source that an installed dbt package defines, or one whose standard test + needs a package the project has not installed, is a singular test (a SQL file to save under + `tests/`) because dbt accepts only one schema entry per resource. +- A `relationships` test names its parent from what the project already declares (relationship tests, + foreign-key constraints), from joins in the models that read the column, and from declared keys, and + is offered only if it verifies on both copies. With no parent that verifies, the report says so. + +`--format json` prints the same result as JSON, including every executed fault (not only the ones +that slipped through), the no-fault controls and per-fault timings. + +!!! note + Findings are gaps in the project's tests. They are not evidence that the data in the warehouse + today is wrong: the corruption only ever exists in the copy. + +### Cost + +One `dbt build --full-refresh` of the models, seeds and snapshots on the copy, then for every +corrupted relation three no-fault control runs, and for every fault one `dbt run` of the downstream +models plus one `dbt test`. All of it is single-threaded. The run above took 158 seconds on a +laptop for 52 faults over 6 relations. The controls are a fixed cost per relation, so a small budget +spread over many relations is dominated by them: on a larger project, 20 faults spread over 17 +relations took 186 seconds, 129 of them in profiling and controls. Use `--model` to concentrate the +budget. + +### Safety + +dbt runs on a copy of the database and a copy of the project, both in a temporary directory (or +`--work-dir`), with its own profile, target path and log path. The directory is removed on success, +failure and interrupt, and the command reports where it was. The project's database is opened only +to copy it, and the command checks afterwards that its size and modification time are unchanged. + +It refuses to run when it can see that dbt would reach beyond that copy: an in-memory or MotherDuck +database, a profile with `attach` or `plugins`, a database with a pending write-ahead log, a model +with the `external` materialization, a relation in another database, or a hook that runs `ATTACH`, +`COPY` or `EXPORT DATABASE`. + +It does not inspect what macros and Python models do. Code there that writes to an absolute path, or +attaches another database by absolute path, would act on the real thing. A `kill -9` leaves the +temporary directory behind. + +### Limits + +- **DuckDB only.** Any other warehouse is refused before anything runs +- The models, seeds and snapshots must build on the copy; a build error stops the run. Tests that + already fail do not: they are reported and cannot catch a fault +- Needs dbt-core (not dbt Fusion) with the project's adapter. Set `ALTIMATE_DBT_PATH` if dbt is not + found +- Needs an `@altimateai/altimate-core` that includes the fault-injection engine; the command says so + when the installed one does not +- Row-level detail (which columns changed in how many rows) needs a declared unique key. Up to 16,384 + rows both versions of a changed model are compared in full. Above that, up to 2,000,000 rows, the + command compares a hash of every row and then reads only the changed rows, which is exact for the + counts of added, removed and changed rows; when more than 20,000 rows changed the per-column + counts come from the first 20,000 of them, and the report says so. A 580,000-row model took about + 7 seconds per fault in a test on a loaded laptop. Larger models keep the row count and checksum. +- A proposed test assumes the data keeps its shape: an accepted-values list fails when a new + legitimate value first appears, and a row-count floor when the model legitimately shrinks. +- Each fault copies the database once. On filesystems without copy-on-write clones that is a full + copy per fault + +--- + ## altimate-dbt CLI `altimate-dbt` is a standalone CLI for dbt workflows. It auto-detects your dbt project directory, Python environment, and adapter type (Snowflake, BigQuery, Databricks, Redshift, etc.). diff --git a/docs/docs/data-engineering/tools/index.md b/docs/docs/data-engineering/tools/index.md index be774c4dcc..224acbc81c 100644 --- a/docs/docs/data-engineering/tools/index.md +++ b/docs/docs/data-engineering/tools/index.md @@ -13,7 +13,7 @@ altimate has 100+ specialized tools organized by function. | [Schema Tools](schema-tools.md) | 7 tools | Inspection, search, PII detection, tagging, diffing | | [FinOps Tools](finops-tools.md) | 8 tools | Cost analysis, warehouse sizing, unused resources, RBAC | | [Lineage Tools](lineage-tools.md) | 1 tool | Column-level lineage tracing with confidence scoring | -| [dbt Tools](dbt-tools.md) | 3 tools + 6 skills | Run, manifest parsing, unit test generation, scaffolding, `altimate-dbt` CLI | +| [dbt Tools](dbt-tools.md) | 4 tools + 6 skills | Run, manifest parsing, unit test generation, fault injection, scaffolding, `altimate-dbt` CLI | | [Warehouse Tools](warehouse-tools.md) | 6 tools | Environment scanning, connection management, discovery, testing | | [Altimate Memory](memory-tools.md) | 3 tools | Persistent cross-session memory for warehouse config, conventions, and preferences | | [Training](../training/index.md) | 3 tools + 3 skills | Correct the agent once, it remembers forever, your team inherits it | diff --git a/docs/docs/usage/cli.md b/docs/docs/usage/cli.md index 876533879b..8d54692b93 100644 --- a/docs/docs/usage/cli.md +++ b/docs/docs/usage/cli.md @@ -28,6 +28,7 @@ altimate --agent analyst | ----------- | ------------------------------ | | `run` | Run a prompt non-interactively | | `check` | Run deterministic SQL checks (no LLM required) -- see [SQL Check](check.md) | +| `fault-injection` | Find the upstream data faults a dbt project's tests miss (no LLM required) -- see [dbt Tools](../data-engineering/tools/dbt-tools.md#dbt_fault_injection) | | `serve` | Start the HTTP API server | | `web` | Start the web UI | | `agent` | Agent management | diff --git a/packages/opencode/src/altimate/native/connections/fault-injection-report.ts b/packages/opencode/src/altimate/native/connections/fault-injection-report.ts new file mode 100644 index 0000000000..43d6a6a51d --- /dev/null +++ b/packages/opencode/src/altimate/native/connections/fault-injection-report.ts @@ -0,0 +1,339 @@ +/** + * Human-readable rendering of a fault-injection result. Shared by the + * `fault-injection` CLI command and the `dbt_fault_injection` tool so both say + * exactly the same thing. + */ + +import type { DbtFaultInjectionResult, FaultInjectionNodeInfo } from "../types" + +type Json = Record + +function plural(n: number, word: string): string { + return `${n} ${word}${n === 1 ? "" : "s"}` +} + +function seconds(ms: number | undefined): string { + if (ms === undefined) return "?" + return ms >= 10_000 ? `${Math.round(ms / 1000)}s` : `${(ms / 1000).toFixed(1)}s` +} + +function nodeLabel(id: string, nodes: Record): string { + const node = nodes[id] + if (!node) return id + if (node.resource_type === "source") return `source ${node.source_name}.${node.name}` + return `${node.resource_type} ${node.name}` +} + +/** What the fault did to the producer, in plain words. */ +function describeFault(result: Json): string { + const rows = `${result.affected_rows} of ${plural(Number(result.producer_rows), "row")}` + const column = `\`${result.column}\`` + switch (result.template) { + case "duplicate_rows": + return `${rows} duplicated` + case "drop_rows": + return `${rows} removed` + case "null_out": + return `${column} set to NULL in ${rows}` + case "unit_scale": + return `${column} multiplied by 100 in ${rows}` + case "category_inject": + return `${column} set to a value never seen before in ${rows}` + case "date_shift": + return `${column} moved one day later in ${rows}` + case "orphan_fk": + return `${column} set to an id that exists nowhere else in ${rows}` + default: + return `${result.template}${result.column ? ` on ${column}` : ""}, ${rows} affected` + } +} + +function describeChange(changed: Json, nodes: Record): string { + const label = nodeLabel(changed.unique_id, nodes) + const before = changed.baseline_rows + const after = changed.rows + const comparison = changed.comparison ?? {} + if (comparison.method === "keyed") { + const parts: string[] = [] + if (comparison.rows_changed > 0) { + const columns = (comparison.columns ?? []).map((c: Json) => `${c.column}: ${c.rows_changed}`).join(", ") + const sampled = + typeof comparison.sampled_rows === "number" + ? `; the per-column counts cover the first ${comparison.sampled_rows} changed rows by key` + : "" + parts.push(`${plural(comparison.rows_changed, "row")} changed${columns ? ` (${columns}${sampled})` : ""}`) + } + if (comparison.rows_added > 0) parts.push(`${plural(comparison.rows_added, "row")} added`) + if (comparison.rows_removed > 0) parts.push(`${plural(comparison.rows_removed, "row")} removed`) + const what = parts.length ? parts.join(", ") : "content changed" + return `${label}: ${what} of ${before} (matched on ${comparison.key_columns.join(", ")})` + } + const why = comparison.note ? `; ${comparison.note}, so no row-level detail` : "" + if (before === null || before === undefined || after === null || after === undefined) { + return `${label}: relation missing on one side (${before ?? "missing"} -> ${after ?? "missing"} rows)` + } + if (before !== after) return `${label}: row count ${before} -> ${after}${why}` + return `${label}: content changed, row count unchanged at ${after}${why}` +} + +function indent(text: string, spaces: number): string { + const pad = " ".repeat(spaces) + return text + .split("\n") + .map((line) => (line ? pad + line : line)) + .join("\n") +} + +/** dbt 1.8 renamed the `tests:` key to `data_tests:`. */ +function testsKey(dbtVersion: string | undefined): string { + const match = dbtVersion?.match(/^(\d+)\.(\d+)/) + if (!match) return "data_tests" + const [major, minor] = [Number(match[1]), Number(match[2])] + return major > 1 || (major === 1 && minor >= 8) ? "data_tests" : "tests" +} + +/** A name as a YAML scalar: plain when that is safe, quoted otherwise. */ +function yamlScalar(value: string): string { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value) ? value : JSON.stringify(value) +} + +/** The proposed test as a block that can be pasted into a schema file. */ +export function proposedTestYaml( + proposal: Json, + nodes: Record, + dbtVersion?: string, +): string { + const node = nodes[proposal.node_id] + const key = testsKey(dbtVersion) + const name = yamlScalar(node?.name ?? String(proposal.node_id).split(".").pop() ?? "") + const lines: string[] = [] + let depth: number + if (proposal.resource_section === "sources" && node?.source_name) { + lines.push("sources:", ` - name: ${yamlScalar(node.source_name)}`, " tables:", ` - name: ${name}`) + depth = 8 + } else { + lines.push(`${proposal.resource_section}:`, ` - name: ${name}`) + depth = 4 + } + if (proposal.column) { + lines.push(indent("columns:", depth), indent(`- name: ${yamlScalar(String(proposal.column))}`, depth + 2)) + depth += 4 + } + lines.push(indent(`${key}:`, depth), indent(String(proposal.yaml), depth + 2)) + return lines.join("\n") +} + +function describeVerification(proposal: Json): string { + const check = proposal.verification + if (!check) return "Not verified against the data." + if (check.catches_fault) { + return `Verified on the data: passes on the clean data and fails on the corrupted copy (${check.sandbox_failures} failing).` + } + // The engine only returns a proposal that passed this check; anything else is shown as unverified. + return "Not verified against the data." +} + +function formatSlipped( + index: number, + result: Json, + nodes: Record, + project: string, + dbtVersion?: string, +): string[] { + const lines: string[] = [] + lines.push(`${index}. ${nodeLabel(result.producer_id, nodes)}: ${describeFault(result)}`) + lines.push(` Fault id: ${result.fault_id}`) + lines.push(` ${plural(result.tests_run, "test")} ran and none failed because of the fault. Changed downstream:`) + for (const changed of result.changed_relations ?? []) { + lines.push(` - ${describeChange(changed, nodes)}`) + } + const proposal = result.proposed_test + if (!proposal) { + lines.push(` No test is proposed for this fault${result.proposal_note ? `: ${result.proposal_note}` : "."}`) + return lines + } + const target = nodes[proposal.node_id] + const fromPackage = target?.package_name && target.package_name !== project ? target.package_name : undefined + if (proposal.form === "singular_sql") { + lines.push(` Proposed test: ${proposal.test}`) + lines.push(` ${describeVerification(proposal)}`) + // The engine says why it chose a file over a schema entry; older engines do not. + const why = + proposal.singular_reason ?? + (fromPackage + ? `The ${proposal.resource_section === "sources" ? "source" : "node"} belongs to the installed package ${fromPackage}, ` + + `and dbt accepts only one description per resource, so a schema entry cannot be added for it.` + : "A schema entry is not possible for this test.") + lines.push(` ${why} This is a singular test instead (plain SQL, no package needed).`) + lines.push(` Save as ${proposal.file} (or in another folder listed under test-paths in dbt_project.yml):`) + lines.push(indent(String(proposal.sql).trimEnd(), 6)) + if (proposal.rationale) lines.push(` Note: ${proposal.rationale}`) + return lines + } + const file = !fromPackage && target?.patch_path ? ` (the node is described in ${target.patch_path})` : "" + lines.push(` Proposed test: ${proposal.test}${file}`) + lines.push(` ${describeVerification(proposal)}`) + if (fromPackage) { + // An engine that predates singular tests proposes a schema entry for a package's node. + const where = target?.patch_path ? ` (${target.patch_path} there)` : "" + lines.push( + proposal.resource_section === "sources" + ? ` The source is defined in the installed package ${fromPackage}${where}, not in this project.` + : ` The node belongs to the installed package ${fromPackage}${where}. dbt accepts only one description ` + + `per node, so this block cannot simply be added to this project's YAML.`, + ) + } + lines.push(indent(proposedTestYaml(proposal, nodes, dbtVersion), 6)) + if (proposal.rationale) lines.push(` Note: ${proposal.rationale}`) + return lines +} + +/** + * A catch rate as a percentage with one decimal, rounded down so that 99.96% + * with a fault still slipping through never reads as 100%. + */ +export function formatRate(rate: number): string { + return `${(Math.floor(rate * 1000) / 10).toFixed(1)}%` +} + +/** The summary's catch rate, or "n/a" when no fault was caught or slipped. */ +export function catchRateText(summary: Json): string { + return typeof summary.catch_rate === "number" ? formatRate(summary.catch_rate) : "n/a" +} + +/** What became of the user's database and of the work directory, as far as the run established. */ +function closingLines(result: DbtFaultInjectionResult): string[] { + const lines: string[] = [] + if (result.database && result.original_unchanged === true) { + lines.push(`${result.database} is unchanged (same size and modification time as before the run).`) + } + if (result.work_dir) { + lines.push( + result.work_dir_removed + ? `The work directory ${result.work_dir} has been removed.` + : `The work directory ${result.work_dir} could not be removed; delete it by hand.`, + ) + } + if (result.engine?.source === "dev-override") { + lines.push(`Engine: development build at ${result.engine.path}.`) + } + return lines +} + +/** One-line outcome, used as the tool title. */ +export function summarizeFaultInjection(result: DbtFaultInjectionResult): string { + if (!result.success || !result.report) return result.interrupted ? "interrupted" : "ERROR" + const s = result.report.summary ?? {} + return `catch rate ${catchRateText(s)} (${s.killed} caught, ${s.slipped_through} slipped through)` +} + +/** The full text report. Every line states something the run observed. */ +export function formatFaultInjection(result: DbtFaultInjectionResult): string { + const lines: string[] = [] + + if (!result.success || !result.report) { + lines.push(result.interrupted ? "Fault injection interrupted." : `Fault injection failed: ${result.error}`) + lines.push(...closingLines(result)) + return lines.join("\n") + } + + const report = result.report + const s = report.summary ?? {} + const nodes = result.nodes ?? {} + const decided = Number(s.killed) + Number(s.slipped_through) + + lines.push(`Fault injection: ${report.project} (${result.warehouse})`) + lines.push("") + lines.push( + decided > 0 + ? `Catch rate: ${catchRateText(s)} (${s.killed} of ${decided} faults that mattered were caught)` + : "Catch rate: n/a (no fault was caught and none changed downstream data)", + ) + lines.push( + ` ${plural(s.executed, "fault")} injected: ${s.killed} caught, ${s.slipped_through} slipped through, ` + + `${s.inert} harmless, ${s.invalid} invalid`, + ) + lines.push(" caught = a test failed or a downstream model broke; slipped through = no test failed and downstream data changed;") + lines.push(" harmless = nothing downstream changed; invalid = the fault touched no row or its sandbox failed") + if (s.selected < s.candidates) { + const limited = result.budget !== undefined && s.selected >= result.budget + lines.push( + ` ${s.selected} of ${s.candidates} candidate faults were selected` + + (limited ? ` (budget ${result.budget}). Raise the budget to run more.` : "."), + ) + } + if (s.skipped > 0) lines.push(` ${plural(s.skipped, "selected fault")} not run; see below.`) + if (report.dialect_verified === false) { + lines.push(` The SQL for ${report.dialect} has not been verified against a real engine.`) + } + + const slipped = (report.slipped_through ?? []) as Json[] + lines.push("") + if (slipped.length === 0) { + lines.push("No fault slipped through.") + } else { + lines.push(`Slipped through (${slipped.length}): the project's tests did not notice these`) + slipped.forEach((entry, i) => { + lines.push("") + lines.push(...formatSlipped(i + 1, entry, nodes, String(report.project), result.dbt?.version)) + }) + } + + const controls = ((report.controls ?? []) as Json[]).filter( + (c) => c.quarantined || c.volatile_relations?.length || c.failing_tests?.length || c.build_errors?.length, + ) + if (controls.length > 0) { + lines.push("") + lines.push("Seen with no fault injected (excluded from the results above):") + for (const control of controls) { + const label = nodeLabel(control.producer_id, nodes) + if (control.quarantined) lines.push(` - ${label}: not tested (${control.reason ?? "unreliable control run"})`) + if (control.volatile_relations?.length) { + lines.push(` - ${label}: these change on every rebuild: ${control.volatile_relations.join(", ")}`) + } + if (control.failing_tests?.length) { + lines.push(` - ${label}: these tests already fail: ${control.failing_tests.join(", ")}`) + } + if (control.build_errors?.length) { + lines.push(` - ${label}: these models already break: ${control.build_errors.join(", ")}`) + } + } + } + + const skipped = (report.skipped ?? []) as Json[] + if (skipped.length > 0) { + lines.push("") + lines.push("Not run:") + for (const entry of skipped) lines.push(` - ${entry.fault_id}: ${entry.reason}`) + } + + const invalid = ((report.results ?? []) as Json[]).filter((r) => r.outcome === "invalid") + if (invalid.length > 0) { + lines.push("") + lines.push("Invalid:") + for (const entry of invalid) { + lines.push(` - ${entry.fault_id}: ${entry.error ?? "the fault touched no row"}`) + } + } + + const warnings = (report.warnings ?? []) as string[] + if (warnings.length > 0) { + lines.push("") + lines.push("Warnings:") + for (const warning of warnings) lines.push(` - ${warning}`) + } + + lines.push("") + lines.push("These are gaps in the project's tests. They are not evidence that the data in the warehouse today is wrong.") + const t = result.timing + if (t) { + const faultMs = Object.values(t.per_fault_ms ?? {}).reduce((sum, ms) => sum + ms, 0) + const faults = Object.keys(t.per_fault_ms ?? {}).length + const parts = [`setup and baseline build ${seconds(t.setup_ms)}`] + if (faults > 0) parts.push(`${plural(faults, "fault")} ${seconds(faultMs)} (${seconds(faultMs / faults)} each on average)`) + if (t.run_ms !== undefined) parts.push(`profiling and no-fault controls ${seconds(Math.max(t.run_ms - faultMs, 0))}`) + lines.push(`Took ${seconds(t.total_ms)}: ${parts.join("; ")}.`) + } + lines.push(...closingLines(result)) + return lines.join("\n") +} diff --git a/packages/opencode/src/altimate/native/connections/fault-injection.ts b/packages/opencode/src/altimate/native/connections/fault-injection.ts new file mode 100644 index 0000000000..6595d63304 --- /dev/null +++ b/packages/opencode/src/altimate/native/connections/fault-injection.ts @@ -0,0 +1,1478 @@ +/** + * Fault-injection orchestrator — runs the cooperative Rust state machine + * (`FaultInjectionSession`) against a sandbox copy of a dbt project's warehouse. + * + * The Rust engine never touches a database or dbt. It emits one action at a + * time — run this SQL, prepare a sandbox, rebuild these models, run these tests + * — and this file performs it and steps the engine with the result. Same shape + * as `data-diff.ts`, with dbt and a sandbox added. + * + * Safety contract: nothing here writes to the user's database, dbt `target/` + * or `logs/`. Everything runs on copies inside a work directory that is removed + * on success, failure and interrupt. + * + * Engine loading: production uses the `@altimateai/altimate-core` package. + * DEVELOPMENT ONLY: set `ALTIMATE_CORE_DEV_PATH` to a locally built + * altimate-core Node binding (the `crates/altimate-core-node` directory, its + * `index.js`, or the `.node` file) to run against an unpublished engine. The + * override is honoured by local and branch builds only; a published release ignores it. + */ + +import * as fs from "fs" +import * as fsp from "fs/promises" +import * as os from "os" +import * as path from "path" +import { spawn } from "child_process" +import { createRequire } from "module" +import type { Connector } from "@altimateai/drivers/types" +import { InstallationChannel, isPublishableChannel } from "@opencode-ai/core/installation/version" +import type { + DbtFaultInjectionParams, + DbtFaultInjectionProgress, + DbtFaultInjectionResult, + FaultInjectionNodeInfo, +} from "../types" + +// --------------------------------------------------------------------------- +// Engine protocol (mirrors crates/altimate-core/src/fault_injection/types.rs) +// --------------------------------------------------------------------------- + +export type SqlTarget = "Baseline" | "Sandbox" + +export interface FaultSqlTask { + id: string + target: SqlTarget + sql: string + expected_shape: "SingleRow" | "RowSet" | "Statement" +} + +export type FaultAction = + | { + type: "ExecuteSql" + id: string + phase: string + producer_id?: string + fault_id?: string + sequential: boolean + tasks: FaultSqlTask[] + } + | { type: "PrepareSandbox"; id: string; purpose: "control" | "fault"; producer_id: string; fault_id?: string } + | { + type: "RebuildNodes" + id: string + producer_id: string + fault_id?: string + node_ids: string[] + select: string[] + exclude: string[] + full_refresh: boolean + } + | { type: "RunTests"; id: string; producer_id: string; fault_id?: string; test_ids: string[]; select: string[] } + | { type: "Done"; report: Record } + | { type: "Error"; message: string } + +export type PerformableAction = Exclude + +export interface FaultSqlResponse { + id: string + rows?: (string | null)[][] + error?: string + relation_missing?: boolean +} + +export interface NodeStatus { + unique_id: string + status: string +} + +export type FaultStepResult = + | { type: "Sql"; id: string; responses: FaultSqlResponse[] } + | { type: "Ok"; id: string } + | { type: "NodeResults"; id: string; results: NodeStatus[] } + | { type: "Failed"; id: string; message: string; timed_out?: boolean } + +export interface RelationColumns { + database?: string + schema: string + name: string + relation_type: "table" | "view" + columns: Array<{ name: string; data_type: string }> +} + +/** The NAPI class, reduced to what the driver calls. JSON strings in and out. */ +export interface FaultInjectionSessionLike { + start(): string + step(resultJson: string): string + report(): string +} + +export type FaultInjectionSessionCtor = new (specJson: string) => FaultInjectionSessionLike + +// --------------------------------------------------------------------------- +// Engine loading +// --------------------------------------------------------------------------- + +/** + * First `@altimateai/altimate-core` release that exports `FaultInjectionSession`. + * The class was added after 0.7.0; update this if the release that ships it is + * numbered differently. + */ +export const FAULT_INJECTION_MIN_CORE_VERSION = "0.8.0" + +/** Development-only override: path to a locally built altimate-core Node binding. */ +export const CORE_DEV_PATH_ENV = "ALTIMATE_CORE_DEV_PATH" + +export interface LoadedEngine { + Session: FaultInjectionSessionCtor + /** "package" in production; "dev-override" when ALTIMATE_CORE_DEV_PATH is set. */ + source: "package" | "dev-override" + path?: string +} + +function installedCoreVersion(): string | undefined { + try { + const require = createRequire(import.meta.url) + return require("@altimateai/altimate-core/package.json").version as string + } catch { + return undefined + } +} + +/** + * Load `FaultInjectionSession`. Throws an Error whose message is meant for the + * user; never crashes the process. + */ +export async function loadFaultInjectionEngine( + env: NodeJS.ProcessEnv = process.env, + // Published releases ("latest", "beta") never honour the override; local and branch builds do. + allowDevOverride: boolean = !isPublishableChannel(InstallationChannel), + importCore: () => Promise = () => import("@altimateai/altimate-core"), +): Promise { + const override = env[CORE_DEV_PATH_ENV] + // Loading a native addon from a path in the environment is for development builds only. + if (override && allowDevOverride) { + const resolved = path.resolve(override) + let entry = resolved + try { + if (fs.statSync(resolved).isDirectory()) entry = path.join(resolved, "index.js") + } catch { + throw new Error(`${CORE_DEV_PATH_ENV} points at "${resolved}", which does not exist.`) + } + let mod: any + try { + mod = createRequire(import.meta.url)(entry) + } catch (e) { + throw new Error(`${CORE_DEV_PATH_ENV}: could not load "${entry}": ${errorText(e)}`) + } + const Session = mod?.FaultInjectionSession + if (typeof Session !== "function") { + throw new Error(`${CORE_DEV_PATH_ENV}: "${entry}" does not export FaultInjectionSession.`) + } + return { Session, source: "dev-override", path: entry } + } + + let core: any + try { + core = await importCore() + } catch (e) { + throw new Error(`altimate-core NAPI module unavailable: ${errorText(e)}`) + } + const Session = core?.FaultInjectionSession ?? core?.default?.FaultInjectionSession + if (typeof Session !== "function") { + const installed = installedCoreVersion() + throw new Error( + `Fault injection needs @altimateai/altimate-core ${FAULT_INJECTION_MIN_CORE_VERSION} or newer` + + ` (the installed version${installed ? `, ${installed},` : ""} has no FaultInjectionSession).` + + ` Upgrade altimate-code to a release that bundles it.` + + (override ? ` ${CORE_DEV_PATH_ENV} is set but a published release does not honour it.` : ""), + ) + } + return { Session, source: "package" } +} + +let engineAvailability: Promise | undefined + +/** + * Whether the engine provides `FaultInjectionSession`. Cached so the native module + * is loaded once; an injected loader bypasses the cache. Never throws. + */ +export function isFaultInjectionEngineAvailable(load?: typeof loadFaultInjectionEngine): Promise { + if (load) return load().then(() => true, () => false) + engineAvailability ??= loadFaultInjectionEngine().then(() => true, () => false) + return engineAvailability +} + +/** Test hook: forget the cached availability result. */ +export function resetFaultInjectionEngineAvailability(): void { + engineAvailability = undefined +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +function errorText(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} + +export class FaultInjectionInterrupted extends Error { + constructor() { + super("interrupted") + this.name = "FaultInjectionInterrupted" + } +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new FaultInjectionInterrupted() +} + +/** + * Render one driver value as exact text. Checksums are HUGEINT and arrive as + * BigInt — `toString()` keeps every digit, `Number()` would not. + */ +export function renderValue(value: unknown): string | null { + if (value === null || value === undefined) return null + if (typeof value === "string") return value + if (typeof value === "bigint") return value.toString() + if (value instanceof Date) return value.toISOString() + return String(value) +} + +const ERROR_TEXT_LIMIT = 500 + +// --------------------------------------------------------------------------- +// Sandbox strategy — one per warehouse +// --------------------------------------------------------------------------- + +/** + * Everything warehouse-specific: how the baseline and the sandbox come to be, + * how SQL reaches each, and how dbt is pointed at them. DuckDB copies a file; + * a cloud warehouse would clone a schema or database. + */ +export interface SandboxStrategy { + /** dbt adapter type this strategy serves. */ + readonly warehouse: string + /** `SqlDialect` name passed to the engine. */ + readonly dialect: string + /** The user's database, which is never written. For messages. */ + readonly original: string + /** + * Create the baseline copy and the dbt profiles for both copies. Must throw + * unless it can establish that everything dbt and this process will write is + * a copy. + */ + setup(): Promise + /** Directory holding the `profiles.yml` that points dbt at `target`. */ + profilesDir(target: SqlTarget): string + /** + * Throw when the parsed project would visibly read or write something other + * than the copied warehouse. Called before anything is built. + */ + assertManifestIsolated(manifest: Record): void + /** Run one statement and return the driver's rows. */ + execute(target: SqlTarget, sql: string): Promise + /** True when the error is a catalog error: a relation the query reads does not exist. */ + isRelationMissing(error: unknown): boolean + /** Columns of every relation in the baseline, in ordinal order. */ + listRelations(): Promise + /** Discard any previous sandbox and make a fresh copy of the baseline. */ + prepareSandbox(): Promise + /** Drop every handle this process holds on `target`, so dbt can open it. */ + release(target: SqlTarget): Promise + /** + * Whether the user's database still looks as it did before the run. "unknown" + * when the run ended before it was first examined. + */ + verifyOriginalUntouched(): Promise<{ status: "unchanged" | "unknown" } | { status: "changed"; detail: string }> + /** Close connections. The work directory is removed by the caller. */ + close(): Promise +} + +export interface SandboxContext { + /** The user's project directory; relative database paths resolve against it. */ + projectDir: string + /** Scratch directory owned by this run. */ + workDir: string + profileName: string + targetName: string + /** The selected output of the user's profile, env_var() already resolved where possible. */ + output: Record + /** The same output exactly as written in profiles.yml. */ + rawOutput: Record +} + +type SandboxFactory = (ctx: SandboxContext) => SandboxStrategy + +const SANDBOX_STRATEGIES: Record = { + duckdb: (ctx) => new DuckDbSandbox(ctx), +} + +/** Warehouses fault injection can sandbox today. */ +export function supportedWarehouses(): string[] { + return Object.keys(SANDBOX_STRATEGIES) +} + +/** Resolve the strategy for a dbt adapter type, or throw before any work is done. */ +export function resolveSandboxFactory(adapterType: string): SandboxFactory { + const factory = SANDBOX_STRATEGIES[adapterType.toLowerCase()] + if (!factory) { + throw new Error( + `Fault injection does not support ${adapterType} yet. It currently works on ` + + `${supportedWarehouses().join(", ")} projects only, because it needs a private copy of the warehouse to corrupt.`, + ) + } + return factory +} + +function sqlString(value: string): string { + return `'${value.replace(/'/g, "''")}'` +} + +function sqlIdent(value: string): string { + return `"${value.replace(/"/g, '""')}"` +} + +/** Profile keys that make dbt-duckdb read or write something other than the one database file. */ +const DUCKDB_UNSAFE_PROFILE_KEYS = ["attach", "plugins", "remote", "is_ducklake"] + +/** + * DuckDB: the baseline and the sandbox are file copies. + * + * Both copies keep the original file name, each in its own directory, because + * DuckDB names the catalog after the file and compiled dbt SQL carries that + * catalog name. + * + * This process never opens a copy directly. Each side has an in-memory DuckDB + * that ATTACHes the file and DETACHes it again before dbt runs. DETACH releases + * the file lock immediately; closing a node-duckdb handle only releases it when + * the handle is garbage collected, which is too late for the dbt process that + * needs the file next. + */ +export class DuckDbSandbox implements SandboxStrategy { + readonly warehouse = "duckdb" + readonly dialect = "duckdb" + readonly original: string + + private readonly fileName: string + private readonly catalog: string + private readonly baselinePath: string + private readonly sandboxPath: string + private readonly connectors: Partial> = {} + private readonly attached: Record = { Baseline: false, Sandbox: false } + private originalStat?: { size: number; mtimeMs: number } + + constructor(private readonly ctx: SandboxContext) { + const configured = ctx.output.path + if (typeof configured !== "string" || configured === "") { + throw new Error(`The dbt target "${ctx.targetName}" has no DuckDB "path"; nothing to copy.`) + } + if (configured.includes("{{")) { + throw new Error( + `Could not resolve the DuckDB path of dbt target "${ctx.targetName}" ("${configured}"). ` + + `Set the environment variables it reads and retry.`, + ) + } + if (configured === ":memory:" || /^(md|motherduck):/i.test(configured) || /^[a-z][a-z0-9+.-]*:\/\//i.test(configured)) { + throw new Error( + `Refusing to run: the dbt target "${ctx.targetName}" uses "${configured}", which is not a local DuckDB file ` + + `that can be copied. Fault injection only runs on a copy.`, + ) + } + for (const key of DUCKDB_UNSAFE_PROFILE_KEYS) { + const value = ctx.rawOutput[key] + const empty = value === undefined || value === null || value === false || (Array.isArray(value) && value.length === 0) + if (!empty) { + throw new Error( + `Refusing to run: the dbt target "${ctx.targetName}" sets "${key}", so dbt would reach data outside ` + + `the database file and fault injection cannot establish that it only touches a copy.`, + ) + } + } + // dbt-duckdb resolves a relative path against the directory dbt runs in. + this.original = path.resolve(ctx.projectDir, configured) + this.fileName = path.basename(this.original) + // dbt-duckdb calls the database after the file (minus its extension) unless the profile names it. + const named = ctx.output.database + this.catalog = typeof named === "string" && named !== "" ? named : this.fileName.replace(/\.[^.]*$/, "") + if (["memory", "temp", "system"].includes(this.catalog.toLowerCase())) { + throw new Error( + `Refusing to run: a DuckDB database called "${this.catalog}" collides with a catalog DuckDB reserves, ` + + `so the copy cannot be attached under the name the compiled SQL uses. Rename ${this.fileName}.`, + ) + } + this.baselinePath = path.join(ctx.workDir, "baseline", this.fileName) + this.sandboxPath = path.join(ctx.workDir, "sandbox", this.fileName) + } + + profilesDir(target: SqlTarget): string { + return path.join(this.ctx.workDir, target === "Baseline" ? "profiles-baseline" : "profiles-sandbox") + } + + assertManifestIsolated(manifest: Record): void { + const enabled = (node: any) => node?.config?.enabled !== false + const nodes = Object.values(manifest.nodes ?? {}).filter(enabled) + const refuse = (what: string, names: string[]) => + new Error( + `Refusing to run: ${what} (${names.slice(0, 5).join(", ")}${names.length > 5 ? ", ..." : ""}). ` + + `Fault injection cannot establish that this would only touch the copy of the database.`, + ) + + const external = nodes.filter((n) => n.resource_type === "model" && n.config?.materialized === "external") + if (external.length > 0) { + throw refuse( + `${external.length} model(s) use the "external" materialization, which writes files outside the database`, + external.map((n) => n.name), + ) + } + + // Everything dbt builds or reads must live in the one database that was copied. + const elsewhere = [...nodes, ...Object.values(manifest.sources ?? {}).filter(enabled)].filter( + (n) => + ["model", "seed", "snapshot", "source"].includes(n.resource_type) && + typeof n.database === "string" && + n.database !== "" && + n.database.toLowerCase() !== this.catalog.toLowerCase(), + ) + if (elsewhere.length > 0) { + throw refuse( + `${elsewhere.length} relation(s) live in a database other than "${this.catalog}"`, + elsewhere.map((n) => `${n.name} in ${n.database}`), + ) + } + + // Hooks and on-run-start/end operations are the usual way to reach outside the database. + const reachesOutside = /\b(attach|copy|export\s+database)\b/i + const hookSql = (n: any): string[] => + [...(n.config?.["pre-hook"] ?? []), ...(n.config?.["post-hook"] ?? [])].map((h: any) => + typeof h === "string" ? h : String(h?.sql ?? ""), + ) + const risky = nodes.filter((n) => + n.resource_type === "operation" + ? reachesOutside.test(String(n.raw_code ?? n.raw_sql ?? "")) + : hookSql(n).some((sql) => reachesOutside.test(sql)), + ) + if (risky.length > 0) { + throw refuse( + `${risky.length} hook(s) run ATTACH, COPY or EXPORT DATABASE, which can read or write files outside the database`, + risky.map((n) => n.name), + ) + } + } + + private copyPath(target: SqlTarget): string { + return target === "Baseline" ? this.baselinePath : this.sandboxPath + } + + async setup(): Promise { + let stat: fs.Stats + try { + stat = await fsp.stat(this.original) + } catch { + throw new Error( + `DuckDB file not found: ${this.original}. Build the project first (dbt build) so there is a database to copy.`, + ) + } + if (!stat.isFile()) throw new Error(`DuckDB path is not a file: ${this.original}`) + if (fs.existsSync(`${this.original}.wal`)) { + throw new Error( + `Refusing to run: ${this.original}.wal exists, so the database has uncommitted changes or is open in ` + + `another process. Close it and retry; a copy taken now could be inconsistent.`, + ) + } + this.originalStat = { size: stat.size, mtimeMs: stat.mtimeMs } + + await fsp.mkdir(path.dirname(this.baselinePath), { recursive: true }) + await fsp.mkdir(path.dirname(this.sandboxPath), { recursive: true }) + await fsp.copyFile(this.original, this.baselinePath, fs.constants.COPYFILE_FICLONE) + await this.assertIsCopy(this.baselinePath) + + for (const target of ["Baseline", "Sandbox"] as const) { + await this.writeProfile(target) + } + + // Fail now, not after a baseline build, if the driver is missing or cannot read the file. + await this.execute("Baseline", "SELECT 1") + await this.release("Baseline") + } + + /** Throw unless `copy` is a distinct file inside the work directory. */ + private async assertIsCopy(copy: string): Promise { + const refuse = (why: string) => + new Error(`Refusing to run: cannot establish that ${copy} is a private copy of ${this.original} (${why}).`) + const [work, realCopy, realOriginal] = await Promise.all([ + fsp.realpath(this.ctx.workDir), + fsp.realpath(copy), + fsp.realpath(this.original), + ]) + if (realCopy === realOriginal) throw refuse("both resolve to the same path") + if (!realCopy.startsWith(work + path.sep)) throw refuse("the copy is outside the work directory") + if (realOriginal.startsWith(work + path.sep)) throw refuse("the original is inside the work directory") + const [a, b] = await Promise.all([fsp.stat(realOriginal), fsp.stat(realCopy)]) + if (a.dev === b.dev && a.ino === b.ino) throw refuse("both are the same file on disk") + } + + /** The user's output with only the path and thread count replaced. */ + private async writeProfile(target: SqlTarget): Promise { + const { default: YAML } = await import("yaml") + const settings = + this.ctx.rawOutput.settings && typeof this.ctx.rawOutput.settings === "object" + ? (this.ctx.rawOutput.settings as Record) + : {} + const configOptions = + this.ctx.rawOutput.config_options && typeof this.ctx.rawOutput.config_options === "object" + ? (this.ctx.rawOutput.config_options as Record) + : undefined + // DuckDB spills to temp_directory under memory pressure. Keep the spill files in the work + // directory (removed on exit) rather than in whatever location the user's profile names. + const tempDir = path.join(this.ctx.workDir, "duckdb-temp", target.toLowerCase()) + const redirect = (o: Record) => ("temp_directory" in o ? { ...o, temp_directory: tempDir } : o) + const output = { + ...this.ctx.rawOutput, + type: "duckdb", + path: this.copyPath(target), + // Single-threaded: row order in rebuilt models must not depend on scheduling. + threads: 1, + settings: redirect({ ...settings, threads: 1 }), + ...(configOptions ? { config_options: redirect(configOptions) } : {}), + } + const dir = this.profilesDir(target) + await fsp.mkdir(dir, { recursive: true }) + await fsp.writeFile( + path.join(dir, "profiles.yml"), + YAML.stringify({ [this.ctx.profileName]: { target: this.ctx.targetName, outputs: { [this.ctx.targetName]: output } } }), + { mode: 0o600 }, + ) + } + + private async connector(target: SqlTarget): Promise { + const existing = this.connectors[target] + if (existing) return existing + const { connect } = await import("@altimateai/drivers/duckdb") + const connector = await connect({ type: "duckdb", path: ":memory:" }) + await connector.connect() + await connector.execute("SET threads = 1") + this.connectors[target] = connector + return connector + } + + /** + * Drop the in-memory instance for `target`. Used when its state is no longer + * known (a failed ATTACH, USE or DETACH); the next statement starts from a + * fresh instance instead of running against the wrong catalog. + */ + private async discard(target: SqlTarget): Promise { + const connector = this.connectors[target] + delete this.connectors[target] + this.attached[target] = false + if (!connector) return + try { + await connector.execute("USE memory") + await connector.execute(`DETACH ${sqlIdent(this.catalog)}`) + } catch { + // already detached, or never attached + } + try { + await connector.close() + } catch { + // best effort + } + } + + async execute(target: SqlTarget, sql: string): Promise { + if (!this.attached[target]) { + const file = this.copyPath(target) + await this.assertIsCopy(file) + const connector = await this.connector(target) + const mode = target === "Baseline" ? " (READ_ONLY)" : "" + try { + await connector.execute(`ATTACH ${sqlString(file)} AS ${sqlIdent(this.catalog)}${mode}`) + await connector.execute(`USE ${sqlIdent(this.catalog)}`) + } catch (e) { + await this.discard(target) + throw e + } + this.attached[target] = true + } + const connector = await this.connector(target) + // noLimit: the driver otherwise appends LIMIT 1001 to every SELECT. + // The driver keys each row by column name, so two result columns with the + // same name would collapse into one. The engine's SQL never repeats an + // expression in a select list; a new engine query must keep that property. + const result = await connector.execute(sql, undefined, undefined, { noLimit: true }) + return result.rows + } + + isRelationMissing(error: unknown): boolean { + // Anywhere at a line start: the driver may prefix DuckDB's text with its own explanation. + return /(^|\n)\s*(Error: )?Catalog Error:/i.test(errorText(error)) + } + + async listRelations(): Promise { + const kinds = new Map() + for (const [schema, table, kind] of await this.execute( + "Baseline", + "SELECT table_schema, table_name, table_type FROM information_schema.tables WHERE table_catalog = current_database()", + )) { + kinds.set(`${schema}\u0000${table}`, kind === "VIEW" ? "view" : "table") + } + const relations = new Map() + for (const [catalog, schema, table, column, dataType] of await this.execute( + "Baseline", + "SELECT table_catalog, table_schema, table_name, column_name, data_type FROM information_schema.columns " + + "WHERE table_catalog = current_database() ORDER BY table_schema, table_name, ordinal_position", + )) { + const key = `${schema}\u0000${table}` + let relation = relations.get(key) + if (!relation) { + relation = { + database: String(catalog), + schema: String(schema), + name: String(table), + relation_type: kinds.get(key) ?? "table", + columns: [], + } + relations.set(key, relation) + } + relation.columns.push({ name: String(column), data_type: String(dataType) }) + } + return [...relations.values()] + } + + async prepareSandbox(): Promise { + await this.release("Sandbox") + await fsp.rm(this.sandboxPath, { force: true }) + await fsp.rm(`${this.sandboxPath}.wal`, { force: true }) + if (fs.existsSync(`${this.baselinePath}.wal`)) { + throw new Error("the baseline copy has a pending write-ahead log; it cannot be copied consistently") + } + await fsp.copyFile(this.baselinePath, this.sandboxPath, fs.constants.COPYFILE_FICLONE) + await this.assertIsCopy(this.sandboxPath) + } + + async release(target: SqlTarget): Promise { + if (!this.attached[target]) return + const connector = this.connectors[target] + if (!connector) return + try { + await connector.execute("USE memory") + await connector.execute(`DETACH ${sqlIdent(this.catalog)}`) + this.attached[target] = false + } catch (e) { + // The file may still be attached and locked. Never reuse this instance. + await this.discard(target) + throw e + } + } + + async verifyOriginalUntouched(): Promise<{ status: "unchanged" | "unknown" } | { status: "changed"; detail: string }> { + if (!this.originalStat) return { status: "unknown" } + try { + const now = await fsp.stat(this.original) + if (now.size !== this.originalStat.size || now.mtimeMs !== this.originalStat.mtimeMs) { + return { status: "changed", detail: `${this.original} changed during the run (its size or modification time differs)` } + } + return { status: "unchanged" } + } catch (e) { + return { status: "changed", detail: `${this.original} could not be re-checked: ${errorText(e)}` } + } + } + + async close(): Promise { + for (const target of ["Baseline", "Sandbox"] as const) { + try { + await this.release(target) + } catch { + // the file is about to be deleted; a failed DETACH changes nothing + } + try { + await this.connectors[target]?.close() + } catch { + // best effort + } + delete this.connectors[target] + } + } +} + +// --------------------------------------------------------------------------- +// dbt +// --------------------------------------------------------------------------- + +export interface DbtOutcome { + exitCode: number | null + /** Per-node status from run_results.json, or null when dbt wrote none. */ + results: NodeStatus[] | null + /** Tail of dbt's output, for error messages. */ + tail: string + timedOut: boolean +} + +export interface DbtRunner { + /** Run `dbt ` against the baseline or sandbox copy. Rejects with FaultInjectionInterrupted on abort. */ + run(target: SqlTarget, args: string[], signal?: AbortSignal): Promise + /** Directory dbt writes `manifest.json` and `run_results.json` to for `target`. */ + targetPath(target: SqlTarget): string +} + +const DEFAULT_DBT_TIMEOUT_MS = 900_000 +const TAIL_BYTES = 4_000 +const KILL_GRACE_MS = 3_000 + +/** + * dbt global flags that can also be set through the environment and that change + * what a rebuild builds, reads or writes. A run must not inherit them. + */ +const DBT_FLAG_ENV = [ + "DBT_DEFER", + "DBT_DEFER_STATE", + "DBT_STATE", + "DBT_FAVOR_STATE", + "DBT_FAIL_FAST", + "DBT_WARN_ERROR", + "DBT_WARN_ERROR_OPTIONS", + "DBT_FULL_REFRESH", + "DBT_EMPTY", + "DBT_SAMPLE", + "DBT_STORE_FAILURES", + "DBT_INDIRECT_SELECTION", + "DBT_RESOURCE_TYPES", + "DBT_EXCLUDE_RESOURCE_TYPES", + "DBT_WRITE_JSON", + "DBT_TARGET", + "DBT_PROFILE", + "DBT_PROJECT_DIR", +] + +/** dbt processes in flight, killed if the process exits under them. */ +const liveChildren = new Set() + +/** Signal dbt and anything it started. dbt runs in its own process group on POSIX. */ +function killTree(child: import("child_process").ChildProcess, signal: NodeJS.Signals): void { + try { + if (process.platform !== "win32" && child.pid) process.kill(-child.pid, signal) + else child.kill(signal) + } catch { + // already gone + } +} + +export interface DbtRunnerOptions { + dbtPath: string + env: Record + /** The directory dbt runs in and reads the project from. */ + projectDir: string + workDir: string + dbtProfile: string + dbtTarget: string + timeoutMs?: number + profilesDir(target: SqlTarget): string +} + +/** dbt through the CLI, with target and log paths inside the work directory. */ +export function createDbtRunner(options: DbtRunnerOptions): DbtRunner { + const timeoutMs = options.timeoutMs ?? DEFAULT_DBT_TIMEOUT_MS + const targetPath = (target: SqlTarget) => + path.join(options.workDir, target === "Baseline" ? "target-baseline" : "target-sandbox") + const logPath = path.join(options.workDir, "logs") + + return { + targetPath, + run(target, args, signal) { + if (signal?.aborted) return Promise.reject(new FaultInjectionInterrupted()) + const [subcommand, ...rest] = args + const profiles = options.profilesDir(target) + const targetDir = targetPath(target) + const resultsFile = path.join(targetDir, "run_results.json") + fs.rmSync(resultsFile, { force: true }) + const argv = [ + "--no-use-colors", + subcommand, + "--project-dir", + options.projectDir, + "--profiles-dir", + profiles, + "--profile", + options.dbtProfile, + "--target", + options.dbtTarget, + "--target-path", + targetDir, + "--log-path", + logPath, + ...rest, + ] + const env: Record = { + ...options.env, + DBT_PROFILES_DIR: profiles, + DBT_TARGET_PATH: targetDir, + DBT_LOG_PATH: logPath, + DBT_SEND_ANONYMOUS_USAGE_STATS: "false", + } + for (const name of DBT_FLAG_ENV) delete env[name] + return new Promise((resolve, reject) => { + const child = spawn(options.dbtPath, argv, { + cwd: options.projectDir, + env, + stdio: ["ignore", "pipe", "pipe"], + // Own process group, so a wrapper script's children are stopped with it. + detached: process.platform !== "win32", + }) + liveChildren.add(child) + let tail = "" + const keep = (chunk: Buffer) => { + tail = (tail + chunk.toString("utf-8")).slice(-TAIL_BYTES) + } + child.stdout?.on("data", keep) + child.stderr?.on("data", keep) + + let timedOut = false + let aborted = false + let killTimer: ReturnType | undefined + const stop = () => { + killTree(child, "SIGTERM") + killTimer = setTimeout(() => killTree(child, "SIGKILL"), KILL_GRACE_MS) + } + const timer = setTimeout(() => { + timedOut = true + stop() + }, timeoutMs) + const onAbort = () => { + aborted = true + stop() + } + signal?.addEventListener("abort", onAbort, { once: true }) + const settle = () => { + clearTimeout(timer) + if (killTimer) clearTimeout(killTimer) + signal?.removeEventListener("abort", onAbort) + liveChildren.delete(child) + } + + child.on("error", (e) => { + settle() + reject(e) + }) + child.on("close", (code) => { + settle() + if (aborted || signal?.aborted) return reject(new FaultInjectionInterrupted()) + if (timedOut) { + return resolve({ exitCode: code, results: null, tail: `dbt ${subcommand} timed out after ${timeoutMs}ms`, timedOut }) + } + let results: NodeStatus[] | null = null + try { + const parsed = JSON.parse(fs.readFileSync(resultsFile, "utf-8")) + results = (parsed.results as Array>).map((r) => ({ + unique_id: String(r.unique_id), + status: String(r.status), + })) + } catch { + results = null + } + resolve({ exitCode: code, results, tail, timedOut }) + }) + }) + }, + } +} + +// --------------------------------------------------------------------------- +// Performing actions +// --------------------------------------------------------------------------- + +export interface PerformDeps { + sandbox: Pick + dbt: Pick + signal?: AbortSignal +} + +function failed(id: string, e: unknown, timedOut = false): FaultStepResult { + return { + type: "Failed", + id, + message: errorText(e).slice(0, ERROR_TEXT_LIMIT), + ...(timedOut ? { timed_out: true } : {}), + } +} + +async function executeSql( + action: Extract, + deps: PerformDeps, +): Promise { + const responses: FaultSqlResponse[] = [] + for (const task of action.tasks) { + throwIfAborted(deps.signal) + try { + const rows = await deps.sandbox.execute(task.target, task.sql) + responses.push({ + id: task.id, + rows: task.expected_shape === "Statement" ? [] : rows.map((row) => row.map(renderValue)), + }) + } catch (e) { + responses.push({ + id: task.id, + error: errorText(e).slice(0, ERROR_TEXT_LIMIT), + // Only a catalog error means "the relation is gone"; anything else is a real failure. + ...(deps.sandbox.isRelationMissing(e) ? { relation_missing: true } : {}), + }) + // A sequential batch stops at the first error; later tasks are omitted. + if (action.sequential) break + } + } + return { type: "Sql", id: action.id, responses } +} + +async function runDbtAction( + id: string, + args: string[], + expected: number, + what: string, + deps: PerformDeps, +): Promise { + try { + // dbt needs the file lock this process may still hold on the sandbox. + await deps.sandbox.release("Sandbox") + } catch (e) { + return failed(id, `could not release the sandbox before ${what}: ${errorText(e)}`) + } + let outcome: DbtOutcome + try { + outcome = await deps.dbt.run("Sandbox", args, deps.signal) + } catch (e) { + if (e instanceof FaultInjectionInterrupted) throw e + return failed(id, e) + } + if (outcome.timedOut) return failed(id, outcome.tail || `${what} timed out`, true) + if (outcome.results === null) { + // dbt wrote no run_results.json. That is only fine when nothing was expected to run. + if (outcome.exitCode !== 0 || expected > 0) { + return failed(id, `${what} produced no results (exit ${outcome.exitCode}): ${outcome.tail.slice(-ERROR_TEXT_LIMIT)}`) + } + return { type: "NodeResults", id, results: [] } + } + return { type: "NodeResults", id, results: outcome.results } +} + +/** Perform one engine action and return the result to step the session with. */ +export async function performAction(action: PerformableAction, deps: PerformDeps): Promise { + throwIfAborted(deps.signal) + switch (action.type) { + case "ExecuteSql": + return executeSql(action, deps) + case "PrepareSandbox": + try { + await deps.sandbox.prepareSandbox() + return { type: "Ok", id: action.id } + } catch (e) { + return failed(action.id, e) + } + case "RebuildNodes": { + // An empty selector would make dbt rebuild the whole project. + if (action.select.length === 0) return { type: "NodeResults", id: action.id, results: [] } + const args = [ + "run", + "--threads", + "1", + ...(action.full_refresh ? ["--full-refresh"] : []), + "--select", + ...action.select, + ...(action.exclude.length > 0 ? ["--exclude", ...action.exclude] : []), + ] + return runDbtAction(action.id, args, action.node_ids.length, "dbt run", deps) + } + case "RunTests": { + if (action.select.length === 0) return { type: "NodeResults", id: action.id, results: [] } + const args = ["test", "--threads", "1", "--select", ...action.select] + return runDbtAction(action.id, args, action.test_ids.length, "dbt test", deps) + } + } +} + +// --------------------------------------------------------------------------- +// The cooperative loop +// --------------------------------------------------------------------------- + +/** A run that needs more actions than this is a bug, not a big project. */ +const MAX_ACTIONS = 1_000_000 + +export interface DriveOptions { + signal?: AbortSignal + onProgress?: (event: DbtFaultInjectionProgress) => void +} + +export interface DriveOutcome { + /** The final report; absent when the engine stopped with an error. */ + report?: Record + error?: string + actions: Record + /** Wall-clock per fault id, over every action that belonged to it. */ + perFaultMs: Record +} + +/** + * Step a session to completion: take an action, perform it, feed the result + * back. Throws FaultInjectionInterrupted when the signal aborts. + */ +export async function driveFaultInjectionSession( + session: FaultInjectionSessionLike, + deps: PerformDeps, + options: DriveOptions = {}, +): Promise { + const actions: Record = {} + const perFaultMs: Record = {} + const started = new Set() + let action = JSON.parse(session.start()) as FaultAction + + for (let steps = 0; steps < MAX_ACTIONS; steps++) { + if (action.type === "Done") return { report: action.report, actions, perFaultMs } + if (action.type === "Error") return { error: action.message ?? "Unknown engine error", actions, perFaultMs } + if (!["ExecuteSql", "PrepareSandbox", "RebuildNodes", "RunTests"].includes(action.type)) { + return { error: `Unexpected action type: ${(action as { type: string }).type}`, actions, perFaultMs } + } + throwIfAborted(options.signal ?? deps.signal) + + actions[action.type] = (actions[action.type] ?? 0) + 1 + if (action.type === "PrepareSandbox" && options.onProgress) { + const key = action.fault_id ?? `control:${action.producer_id}` + if (!started.has(key)) { + started.add(key) + const summary = (JSON.parse(session.report()) as Record).summary ?? {} + options.onProgress( + action.fault_id + ? { + kind: "fault", + fault_id: action.fault_id, + index: [...started].filter((k) => !k.startsWith("control:")).length, + total: Number(summary.selected ?? 0), + } + : { kind: "control", producer_id: action.producer_id }, + ) + } + } + + const began = Date.now() + const result = await performAction(action, deps) + if (action.fault_id) perFaultMs[action.fault_id] = (perFaultMs[action.fault_id] ?? 0) + (Date.now() - began) + action = JSON.parse(session.step(JSON.stringify(result))) as FaultAction + } + return { error: `Exceeded ${MAX_ACTIONS} actions; the session is not converging.`, actions, perFaultMs } +} + +// --------------------------------------------------------------------------- +// dbt project and profile +// --------------------------------------------------------------------------- + +/** Resolve `{{ env_var('NAME') }}` and `{{ env_var('NAME', 'default') }}`; anything else is left as written. */ +function resolveEnvVars(value: unknown, env: NodeJS.ProcessEnv): unknown { + if (typeof value === "string") { + return value.replace( + /\{\{\s*env_var\s*\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"]\s*)?\)\s*\}\}/g, + (whole, name: string, fallback?: string) => env[name] ?? fallback ?? whole, + ) + } + if (Array.isArray(value)) return value.map((v) => resolveEnvVars(v, env)) + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, resolveEnvVars(v, env)])) + } + return value +} + +export interface DbtTargetInfo { + profileName: string + targetName: string + adapterType: string + output: Record + rawOutput: Record + profilesFile: string +} + +/** Read the profile and target a dbt project would use, in dbt's lookup order. */ +export async function readDbtTarget( + projectDir: string, + options: { profilesDir?: string; target?: string; env?: NodeJS.ProcessEnv } = {}, +): Promise { + const env = options.env ?? process.env + const { default: YAML } = await import("yaml") + const projectFile = path.join(projectDir, "dbt_project.yml") + if (!fs.existsSync(projectFile)) { + throw new Error(`No dbt_project.yml in ${projectDir}. Run this inside a dbt project or pass its directory.`) + } + const project = YAML.parse(await fsp.readFile(projectFile, "utf-8")) as Record | null + const profileName = String(env.DBT_PROFILE || resolveEnvVars(project?.profile ?? "", env)) + if (!profileName) throw new Error(`${projectFile} does not name a profile.`) + + const candidates = [ + options.profilesDir, + env.DBT_PROFILES_DIR, + projectDir, + path.join(os.homedir(), ".dbt"), + ].filter((d): d is string => Boolean(d)) + const profilesFile = candidates + .map((dir) => path.join(path.resolve(dir), "profiles.yml")) + .find((file) => fs.existsSync(file)) + if (!profilesFile) { + throw new Error(`No profiles.yml found (looked in ${candidates.join(", ")}).`) + } + const profiles = YAML.parse(await fsp.readFile(profilesFile, "utf-8")) as Record | null + const profile = profiles?.[profileName] + if (!profile || typeof profile !== "object") { + throw new Error(`Profile "${profileName}" is not defined in ${profilesFile}.`) + } + const targetName = String(options.target ?? (env.DBT_TARGET || resolveEnvVars(profile.target ?? "default", env))) + const rawOutput = profile.outputs?.[targetName] + if (!rawOutput || typeof rawOutput !== "object") { + throw new Error(`Target "${targetName}" is not defined for profile "${profileName}" in ${profilesFile}.`) + } + const output = resolveEnvVars(rawOutput, env) as Record + const adapterType = String(output.type ?? "") + if (!adapterType) throw new Error(`Target "${targetName}" in ${profilesFile} has no "type".`) + return { profileName, targetName, adapterType, output, rawOutput, profilesFile } +} + +/** Map a model, seed, snapshot or source name to the unique ids the engine calls producers. */ +export function resolveProducers(manifest: Record, name: string): string[] { + const wanted = name.trim() + const ids: string[] = [] + for (const [id, node] of Object.entries(manifest.nodes ?? {})) { + if (!["model", "seed", "snapshot"].includes(node?.resource_type)) continue + if (id === wanted || node.name === wanted) ids.push(id) + } + for (const [id, source] of Object.entries(manifest.sources ?? {})) { + if (id === wanted || source.name === wanted || `${source.source_name}.${source.name}` === wanted) ids.push(id) + } + return ids +} + +/** Names and file locations of the nodes a report refers to, for rendering. */ +function collectNodeInfo(manifest: Record, report: Record): Record { + const ids = new Set() + for (const result of (report.results ?? []) as Array>) { + ids.add(result.producer_id) + if (result.proposed_test?.node_id) ids.add(result.proposed_test.node_id) + for (const changed of result.changed_relations ?? []) ids.add(changed.unique_id) + for (const id of [...(result.failed_tests ?? []), ...(result.build_errors ?? [])]) ids.add(id) + } + for (const control of (report.controls ?? []) as Array>) ids.add(control.producer_id) + const info: Record = {} + for (const id of ids) { + const node = manifest.nodes?.[id] ?? manifest.sources?.[id] + if (!node) continue + info[id] = { + name: String(node.name), + resource_type: String(node.resource_type), + ...(node.source_name ? { source_name: String(node.source_name) } : {}), + ...(node.package_name ? { package_name: String(node.package_name) } : {}), + ...(typeof node.patch_path === "string" ? { patch_path: node.patch_path.replace(/^[^:]*:\/\//, "") } : {}), + ...(typeof node.original_file_path === "string" ? { original_file_path: node.original_file_path } : {}), + } + } + return info +} + +// --------------------------------------------------------------------------- +// Work directory lifetime +// --------------------------------------------------------------------------- + +/** Work directories of runs in flight, removed synchronously if the process exits under them. */ +const liveWorkDirs = new Set() +let exitHookInstalled = false + +function trackWorkDir(dir: string): void { + liveWorkDirs.add(dir) + if (exitHookInstalled) return + exitHookInstalled = true + process.on("exit", () => { + for (const child of liveChildren) killTree(child, "SIGKILL") + for (const live of liveWorkDirs) { + try { + fs.rmSync(live, { recursive: true, force: true }) + } catch { + // nothing more can be done while exiting + } + } + }) +} + +async function removeWorkDir(dir: string): Promise { + try { + await fsp.rm(dir, { recursive: true, force: true }) + } catch { + // reported through the return value + } + const removed = !fs.existsSync(dir) + // Left in the set when removal failed, so the exit hook tries once more. + if (removed) liveWorkDirs.delete(dir) + return removed +} + +/** Top-level entries of a project that dbt does not need and that must not be copied. */ +const PROJECT_COPY_SKIP = new Set([".git", "target", "logs", "node_modules", "profiles.yml", ".user.yml"]) + +/** + * Copy the project into the work directory. dbt then runs in the copy, so + * nothing it writes with a relative path (target/, logs/, a hook's export) + * can land in the user's project. + */ +async function copyProject(projectDir: string, dest: string, workDir: string): Promise { + const [root, work] = await Promise.all([fsp.realpath(projectDir), fsp.realpath(workDir)]) + await fsp.cp(root, dest, { + recursive: true, + mode: fs.constants.COPYFILE_FICLONE, + filter: (source) => { + if (source === work || source.startsWith(work + path.sep)) return false + const relative = path.relative(root, source) + if (relative === "") return true + if (!relative.includes(path.sep) && PROJECT_COPY_SKIP.has(relative)) return false + if (/\.(duckdb|wal)$/i.test(source)) return false + // A Python virtualenv, whatever it is called. + if (fs.existsSync(path.join(source, "pyvenv.cfg"))) return false + return true + }, + }) +} + +// --------------------------------------------------------------------------- +// Main orchestrator +// --------------------------------------------------------------------------- + +/** Test seams. Production passes none. */ +export interface FaultInjectionDeps { + loadEngine?: typeof loadFaultInjectionEngine + resolveDbt?: (projectDir: string) => Promise<{ path: string; version: string; env: Record }> +} + +async function resolveDbtBinary( + projectDir: string, +): Promise<{ path: string; version: string; env: Record }> { + const { resolveDbt, validateDbt, buildDbtEnv } = await import("../../../../../dbt-tools/src/dbt-resolve") + const resolved = resolveDbt(undefined, projectDir) + const valid = validateDbt(resolved) + if (!valid) { + throw new Error( + `dbt is not runnable (tried ${resolved.path}, found via ${resolved.source}). ` + + `Install dbt-core with the project's adapter, or set ALTIMATE_DBT_PATH to the dbt executable.`, + ) + } + if (valid.isFusion) { + throw new Error(`Fault injection needs dbt-core; ${resolved.path} is dbt Fusion. Set ALTIMATE_DBT_PATH to a dbt-core executable.`) + } + return { path: resolved.path, version: valid.version, env: buildDbtEnv(resolved) } +} + +function describeDbtFailure(what: string, outcome: DbtOutcome): string { + const bad = (outcome.results ?? []).filter((r) => ["error", "fail", "runtime error"].includes(r.status)) + const nodes = bad.length + ? ` ${bad.length} node(s) failed: ${bad + .slice(0, 8) + .map((r) => `${r.unique_id} (${r.status})`) + .join(", ")}${bad.length > 8 ? ", ..." : ""}.` + : "" + const tail = outcome.tail.trim().split("\n").slice(-12).join("\n") + return `${what} (exit ${outcome.exitCode}).${nodes}${tail ? `\n${tail}` : ""}` +} + +export async function runFaultInjection( + params: DbtFaultInjectionParams, + deps: FaultInjectionDeps = {}, +): Promise { + const startedAt = Date.now() + const signal = params.signal + const progress = (event: DbtFaultInjectionProgress) => { + try { + params.on_progress?.(event) + } catch { + // progress reporting must never break the run + } + } + const stage = (message: string) => progress({ kind: "stage", message }) + const projectDir = path.resolve(params.project_dir ?? process.cwd()) + const fail = (error: string, extra: Partial = {}): DbtFaultInjectionResult => ({ + success: false, + error, + project_dir: projectDir, + ...extra, + }) + + // Everything that can be checked without copying or building is checked first. + let target: DbtTargetInfo + let factory: SandboxFactory + let engine: LoadedEngine + let dbt: { path: string; version: string; env: Record } + try { + target = await readDbtTarget(projectDir, { profilesDir: params.profiles_dir, target: params.target }) + factory = resolveSandboxFactory(target.adapterType) + engine = await (deps.loadEngine ?? loadFaultInjectionEngine)() + dbt = await (deps.resolveDbt ?? resolveDbtBinary)(projectDir) + } catch (e) { + return fail(errorText(e)) + } + + let workDir: string + try { + const parent = path.resolve(params.work_dir ?? os.tmpdir()) + await fsp.mkdir(parent, { recursive: true }) + workDir = await fsp.mkdtemp(path.join(parent, "altimate-fault-injection-")) + trackWorkDir(workDir) + } catch (e) { + return fail(`Could not create a work directory: ${errorText(e)}`) + } + + let sandbox: SandboxStrategy | undefined + const base: Partial = { + project_dir: projectDir, + warehouse: target.adapterType.toLowerCase(), + work_dir: workDir, + dbt: { path: dbt.path, version: dbt.version }, + engine: { source: engine.source, ...(engine.path ? { path: engine.path } : {}) }, + } + let result: DbtFaultInjectionResult + + try { + sandbox = factory({ + projectDir, + workDir, + profileName: target.profileName, + targetName: target.targetName, + output: target.output, + rawOutput: target.rawOutput, + }) + base.database = sandbox.original + stage(`Copying ${sandbox.original} and the project to ${workDir}`) + await sandbox.setup() + throwIfAborted(signal) + const projectCopy = path.join(workDir, "project") + await copyProject(projectDir, projectCopy, workDir) + throwIfAborted(signal) + + const runner = createDbtRunner({ + dbtPath: dbt.path, + env: dbt.env, + projectDir: projectCopy, + workDir, + dbtProfile: target.profileName, + dbtTarget: target.targetName, + timeoutMs: params.dbt_timeout_ms, + profilesDir: (t) => sandbox!.profilesDir(t), + }) + const readManifest = async () => + JSON.parse(await fsp.readFile(path.join(runner.targetPath("Baseline"), "manifest.json"), "utf-8")) as Record + /** A dbt step of the setup. An interrupt or a timeout is reported as such, not as a project failure. */ + const setupDbt = async (args: string[]): Promise => { + const outcome = await runner.run("Baseline", args, signal) + throwIfAborted(signal) + if (outcome.timedOut) throw new Error(`${outcome.tail}. Raise the dbt timeout or run on a smaller project.`) + return outcome + } + + // Parse before anything is built: a project that visibly reaches outside the database is refused. + stage("Parsing the project") + const parsed = await setupDbt(["parse"]) + if (parsed.exitCode !== 0) throw new Error(describeDbtFailure("The project does not parse: dbt parse failed", parsed)) + sandbox.assertManifestIsolated(await readManifest()) + + // Rebuild the baseline copy now, the way every sandbox is rebuilt later: + // from scratch. Models that read the clock, and incremental models, would + // otherwise differ from the sandbox with no fault injected. Tests are left + // out: they change no data, and a failing one would make dbt skip the + // models below it. The control runs find tests that already fail. + stage("Building the baseline copy (dbt build)") + const buildStarted = Date.now() + const built = await setupDbt([ + "build", + "--threads", + "1", + "--full-refresh", + "--exclude", + "resource_type:test", + "resource_type:unit_test", + ]) + const baselineBuildMs = Date.now() - buildStarted + if (built.results === null) { + throw new Error(describeDbtFailure("The project does not build: dbt build wrote no results", built)) + } + if (built.results.some((r) => ["error", "runtime error", "fail", "skipped"].includes(r.status))) { + throw new Error( + describeDbtFailure("The project does not build on a clean copy, so there is no baseline to compare against", built), + ) + } + + stage("Compiling the project") + const compiled = await setupDbt(["compile", "--threads", "1"]) + if (compiled.exitCode !== 0) throw new Error(describeDbtFailure("dbt compile failed", compiled)) + const manifest = await readManifest() + sandbox.assertManifestIsolated(manifest) + + const config: Record = { budget: params.budget ?? 20 } + if (params.seed !== undefined) config.seed = params.seed + if (params.model) { + const producers = resolveProducers(manifest, params.model) + if (producers.length === 0) { + throw new Error(`No model, seed, snapshot or source named "${params.model}" in this project.`) + } + config.producers = producers + } + + const relations = await sandbox.listRelations() + let session: FaultInjectionSessionLike + try { + session = new engine.Session(JSON.stringify({ manifest, relations, dialect: sandbox.dialect, config })) + } catch (e) { + throw new Error(`Failed to create FaultInjectionSession: ${errorText(e)}`) + } + + stage("Injecting faults") + const runStarted = Date.now() + const outcome = await driveFaultInjectionSession( + session, + { sandbox, dbt: runner, signal }, + { signal, onProgress: progress }, + ) + const timing = { + total_ms: 0, + baseline_build_ms: baselineBuildMs, + setup_ms: runStarted - startedAt, + run_ms: Date.now() - runStarted, + per_fault_ms: outcome.perFaultMs, + actions: outcome.actions, + } + if (!outcome.report) { + result = fail(outcome.error ?? "Unknown engine error", { ...base, timing }) + } else { + const report = outcome.report + result = { + ...base, + success: true, + budget: Number(config.budget), + report, + nodes: collectNodeInfo(manifest, report), + timing, + } + } + } catch (e) { + result = + e instanceof FaultInjectionInterrupted + ? fail("Interrupted before the run finished.", { ...base, interrupted: true }) + : fail(errorText(e), base) + } + + // Cleanup runs on success, failure and interrupt alike. + if (sandbox) { + try { + await sandbox.close() + } catch { + // the work directory is removed regardless + } + const original = await sandbox.verifyOriginalUntouched() + if (original.status === "unchanged") result.original_unchanged = true + if (original.status === "changed") { + result.original_unchanged = false + result.success = false + result.error = + `${original.detail}. Fault injection only writes to copies; check what else was using the file.` + + (result.error ? ` (Run error: ${result.error})` : "") + } + } + result.work_dir_removed = await removeWorkDir(workDir) + if (result.timing) result.timing.total_ms = Date.now() - startedAt + else result.timing = { total_ms: Date.now() - startedAt } + return result +} diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index 808e2be75e..cfd80bd005 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -11,6 +11,7 @@ import * as Registry from "./registry" import { discoverContainers } from "./docker-discovery" import { parseDbtProfiles } from "./dbt-profiles" import { runDataDiff } from "./data-diff" +import { runFaultInjection } from "./fault-injection" import type { SqlExecuteParams, SqlExecuteResult, @@ -32,6 +33,8 @@ import type { DbtProfilesResult, DataDiffParams, DataDiffResult, + DbtFaultInjectionParams, + DbtFaultInjectionResult, } from "../types" import type { ConnectionConfig } from "@altimateai/drivers" import { Telemetry } from "../../../telemetry" @@ -770,6 +773,11 @@ register("data.diff", async (params: DataDiffParams): Promise => return runDataDiff(params) }) +// --- dbt.fault_injection --- +register("dbt.fault_injection", async (params: DbtFaultInjectionParams): Promise => { + return runFaultInjection(params) +}) + } // end registerAll // Auto-register on module load diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index f69f72e712..36673040ec 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -1181,6 +1181,85 @@ export interface DataDiffResult { excluded_audit_columns?: string[] } +// --- dbt Fault Injection --- + +/** Progress events emitted while a fault-injection run is in flight. */ +export type DbtFaultInjectionProgress = + | { kind: "stage"; message: string } + | { kind: "control"; producer_id: string } + | { kind: "fault"; fault_id: string; index: number; total: number } + +export interface DbtFaultInjectionParams { + /** dbt project root (the directory holding dbt_project.yml). Defaults to the working directory. */ + project_dir?: string + /** Directory holding profiles.yml. Defaults to dbt's own lookup order. */ + profiles_dir?: string + /** dbt target name. Defaults to the profile's default target. */ + target?: string + /** Corrupt only this model, seed, snapshot or source (name or unique_id). */ + model?: string + /** Maximum number of faults to execute. Default 20. */ + budget?: number + /** Seed of the deterministic fault selection. */ + seed?: number + /** Parent directory for the scratch copies. Defaults to the system temp directory. */ + work_dir?: string + /** Timeout for one dbt invocation, in milliseconds. Default 15 minutes. */ + dbt_timeout_ms?: number + /** In-process only: aborts the run and triggers cleanup. */ + signal?: AbortSignal + /** In-process only: progress callback. */ + on_progress?: (event: DbtFaultInjectionProgress) => void +} + +/** What the report's unique ids refer to, for rendering. */ +export interface FaultInjectionNodeInfo { + name: string + resource_type: string + source_name?: string + /** dbt package the node is defined in; differs from the project name for installed packages. */ + package_name?: string + /** YAML file that already describes the node, relative to its package root, when there is one. */ + patch_path?: string + original_file_path?: string +} + +export interface DbtFaultInjectionResult { + success: boolean + error?: string + /** True when the run was stopped by a signal. */ + interrupted?: boolean + project_dir?: string + /** dbt adapter type of the project's target. */ + warehouse?: string + /** The project's database. Read once to make the copies; never written. */ + database?: string + /** + * True when the database had the same size and modification time after the + * run as before it; false when it did not. Absent when the run ended before + * the database was first examined. + */ + original_unchanged?: boolean + /** Where the copies lived for the duration of the run. */ + work_dir?: string + /** True when the work directory no longer exists. */ + work_dir_removed?: boolean + dbt?: { path: string; version: string } + engine?: { source: "package" | "dev-override"; path?: string } + budget?: number + /** The engine's FaultReport (see altimate-core `fault_injection::FaultReport`). */ + report?: Record + nodes?: Record + timing?: { + total_ms: number + setup_ms?: number + baseline_build_ms?: number + run_ms?: number + per_fault_ms?: Record + actions?: Record + } +} + // --- Method registry --- export const BridgeMethods = { @@ -1227,6 +1306,8 @@ export const BridgeMethods = { "local.test": {} as { params: LocalTestParams; result: LocalTestResult }, // --- data diff --- "data.diff": {} as { params: DataDiffParams; result: DataDiffResult }, + // --- dbt fault injection --- + "dbt.fault_injection": {} as { params: DbtFaultInjectionParams; result: DbtFaultInjectionResult }, // --- altimate-core (existing) --- "altimate_core.validate": {} as { params: AltimateCoreValidateParams; result: AltimateCoreResult }, "altimate_core.lint": {} as { params: AltimateCoreLintParams; result: AltimateCoreResult }, diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts new file mode 100644 index 0000000000..1dafc629f3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -0,0 +1,99 @@ +import z from "zod" +import path from "path" +import { Tool } from "../../tool/tool" +import { Dispatcher } from "../native" +import { formatFaultInjection, summarizeFaultInjection } from "../native/connections/fault-injection-report" + +export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { + description: [ + "Find the upstream data faults a dbt project's own tests would miss.", + "", + "Corrupts one upstream relation at a time in a private copy of the database (duplicated rows, dropped rows,", + "NULLs, values off by 100x, unseen categories, shifted dates, orphaned foreign keys), rebuilds every model", + "downstream of it, and runs the project's tests. Reports the catch rate and, for each fault that slipped", + "through (no test failed but downstream data changed), which downstream models changed and a dbt test that", + "passes on the clean data and fails on the corrupted copy (a schema entry to paste, or a singular test file", + "for resources a dbt package defines), or the reason no stable test exists. Deterministic: no model is involved.", + "", + "Use it when asked how good a dbt project's tests are, what a model's tests would miss, or which tests to add.", + "Do not use it to check whether existing data is correct: findings are test gaps, not data errors.", + "", + "Cost: one `dbt build` of the whole project on the copy, then one `dbt run` of the downstream models plus", + "one `dbt test` per fault, and three more per corrupted relation as a no-fault control. A 20-fault run takes", + "minutes even on a small project. Keep `budget` small, or set `model` to focus on one relation.", + "", + "dbt runs on copies of the database and of the project, which are deleted when the run ends. A project that", + "visibly reaches outside its database (attached databases, external materializations, hooks that ATTACH or", + "COPY) is refused; what macros and Python models do is not inspected.", + "DuckDB projects only for now; any other warehouse is refused before anything runs.", + ].join("\n"), + parameters: z.object({ + project_dir: z + .string() + .optional() + .describe("dbt project root, the directory containing dbt_project.yml. Defaults to the working directory."), + model: z + .string() + .optional() + .describe( + "Corrupt only this model, seed, snapshot or source, and check what its downstream tests catch. " + + "Omit to spread the budget over every relation that has downstream models.", + ), + budget: z + .number() + .int() + .min(1) + .max(500) + .optional() + .default(20) + .describe("Maximum number of faults to inject. Each one rebuilds the downstream models and runs their tests."), + target: z.string().optional().describe("dbt target name. Defaults to the profile's default target."), + profiles_dir: z.string().optional().describe("Directory containing profiles.yml. Defaults to dbt's lookup order."), + }), + async execute(args, ctx) { + const projectDir = path.resolve(args.project_dir ?? process.cwd()) + // This runs dbt, which executes the project's own code. Ask as for any other command. + const command = `dbt build --project-dir ${projectDir}` + await ctx.ask({ + permission: "bash", + patterns: [command], + always: [command], + metadata: { project_dir: projectDir, budget: args.budget, model: args.model }, + }) + + try { + const result = await Dispatcher.call("dbt.fault_injection", { + project_dir: projectDir, + model: args.model, + budget: args.budget, + target: args.target, + profiles_dir: args.profiles_dir, + signal: ctx.abort, + }) + const summary = result.report?.summary + return { + title: `Fault injection: ${summarizeFaultInjection(result)}`, + metadata: { + success: result.success, + ...(summary + ? { + executed: summary.executed as number, + killed: summary.killed as number, + slipped_through: summary.slipped_through as number, + catch_rate: summary.catch_rate as number | null, + } + : {}), + ...(result.error ? { error: result.error } : {}), + }, + output: formatFaultInjection(result), + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { + title: "Fault injection: ERROR", + metadata: { success: false, error: msg }, + output: `Fault injection failed: ${msg}`, + } + } + }, +}) diff --git a/packages/opencode/src/cli/cmd/fault-injection.ts b/packages/opencode/src/cli/cmd/fault-injection.ts new file mode 100644 index 0000000000..638a24fabb --- /dev/null +++ b/packages/opencode/src/cli/cmd/fault-injection.ts @@ -0,0 +1,152 @@ +// altimate_change start — fault-injection: deterministic dbt test-gap CLI command (no LLM required) +import type { Argv } from "yargs" +import { cmd } from "./cmd" +import { Dispatcher } from "../../altimate/native" +import { formatFaultInjection, formatRate } from "../../altimate/native/connections/fault-injection-report" +import type { DbtFaultInjectionProgress } from "../../altimate/native/types" + +const EXIT_INTERRUPTED = 130 + +function progressLine(event: DbtFaultInjectionProgress): string { + switch (event.kind) { + case "stage": + return event.message + case "control": + return `Control run (no fault): ${event.producer_id}` + case "fault": + return `Fault ${event.index}/${event.total}: ${event.fault_id}` + } +} + +export const FaultInjectionCommand = cmd({ + command: "fault-injection [project]", + describe: "find the upstream data faults a dbt project's tests miss (deterministic, no LLM required)", + builder: (yargs: Argv) => + yargs + .positional("project", { + describe: "dbt project directory (default: current directory)", + type: "string", + }) + .option("budget", { + describe: "maximum number of faults to inject; each one rebuilds the downstream models and runs their tests", + type: "number", + default: 20, + }) + .option("model", { + describe: "corrupt only this model, seed, snapshot or source", + type: "string", + }) + .option("target", { + describe: "dbt target name (default: the profile's default target)", + type: "string", + }) + .option("profiles-dir", { + describe: "directory containing profiles.yml (default: dbt's lookup order)", + type: "string", + }) + .option("seed", { + describe: "seed for the deterministic fault selection", + type: "number", + }) + .option("work-dir", { + describe: "parent directory for the temporary database copies (default: the system temp directory)", + type: "string", + }) + .option("format", { + describe: "output format", + choices: ["text", "json"] as const, + default: "text" as const, + }) + .option("fail-under", { + describe: "exit 1 if the catch rate is below this percentage (0-100)", + type: "number", + }), + + handler: async (args: { + project?: string + budget?: number + model?: string + target?: string + "profiles-dir"?: string + profilesDir?: string + seed?: number + "work-dir"?: string + workDir?: string + format?: "text" | "json" + "fail-under"?: number + failUnder?: number + }) => { + const budget = args.budget ?? 20 + if (!Number.isInteger(budget) || budget < 1) { + console.error("Error: --budget must be a positive integer.") + process.exitCode = 1 + return + } + const failUnder = args["fail-under"] ?? args.failUnder + if (failUnder !== undefined && !(failUnder >= 0 && failUnder <= 100)) { + console.error("Error: --fail-under must be a percentage between 0 and 100.") + process.exitCode = 1 + return + } + + if (args.seed !== undefined && !(Number.isSafeInteger(args.seed) && args.seed >= 0)) { + console.error("Error: --seed must be a non-negative integer.") + process.exitCode = 1 + return + } + + // An interrupt aborts the run; the driver then stops dbt and removes its copies before + // returning. A second one does not wait: process.exit runs the driver's exit hook, which + // kills dbt and removes the work directory synchronously. + const controller = new AbortController() + const onSignal = () => { + if (controller.signal.aborted) process.exit(EXIT_INTERRUPTED) + console.error("Interrupted; stopping dbt and removing the work directory...") + controller.abort() + } + const signals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP"] + for (const name of signals) process.on(name, onSignal) + + try { + const result = await Dispatcher.call("dbt.fault_injection", { + project_dir: args.project, + model: args.model, + budget, + target: args.target, + profiles_dir: args["profiles-dir"] ?? args.profilesDir, + seed: args.seed, + work_dir: args["work-dir"] ?? args.workDir, + signal: controller.signal, + on_progress: (event) => console.error(progressLine(event)), + }) + + if (args.format === "json") { + process.stdout.write(JSON.stringify(result, null, 2) + "\n") + } else if (result.success) { + process.stdout.write(formatFaultInjection(result) + "\n") + } else { + console.error(formatFaultInjection(result)) + } + + // process.exitCode rather than process.exit(), so index.ts can flush telemetry. + if (result.interrupted) { + process.exitCode = EXIT_INTERRUPTED + } else if (!result.success) { + process.exitCode = 1 + } else if (failUnder !== undefined) { + const rate = result.report?.summary?.catch_rate + if (typeof rate !== "number") { + // No fault was caught and none slipped through: nothing was measured, so the gate cannot pass. + console.error(`There is no catch rate to compare with --fail-under ${failUnder}.`) + process.exitCode = 1 + } else if (rate * 100 < failUnder) { + console.error(`Catch rate ${formatRate(rate)} is below --fail-under ${failUnder}.`) + process.exitCode = 1 + } + } + } finally { + for (const name of signals) process.removeListener(name, onSignal) + } + }, +}) +// altimate_change end diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 77b419f868..b80f4881dc 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -45,6 +45,9 @@ import { SkillCommand } from "./cli/cmd/skill" // altimate_change start — check: deterministic SQL check command import { CheckCommand } from "./cli/cmd/check" // altimate_change end +// altimate_change start — fault-injection: deterministic dbt test-gap command +import { FaultInjectionCommand } from "./cli/cmd/fault-injection" +// altimate_change end // altimate_change start — link: workspace-binding subcommand import { LinkCommand } from "./cli/cmd/link" import { pilotOffCommand } from "./cli/cmd/workspace-pilot" @@ -79,6 +82,7 @@ const CLI_COMMAND_NAMES = new Set([ "acp", "mcp", "attach", "run", "generate", "debug", "console", "providers", "auth", "agent", "upgrade", "uninstall", "serve", "web", "models", "stats", "export", "import", "github", "gitlab", "review", "pr", "session", "plugin", "plug", "db", "trace", "recap", "skill", "check", "completion", + "fault-injection", // registered conditionally below (workspace / local-install builds) "link", "workspace-serve", ]) @@ -208,6 +212,9 @@ let cli = yargs(args) // altimate_change start — check: register deterministic SQL check command .command(CheckCommand) // altimate_change end + // altimate_change start — fault-injection: register deterministic dbt test-gap command + .command(FaultInjectionCommand) + // altimate_change end // altimate_change start — link: gated on Flag.ALTIMATE_WORKSPACE (pilot) // so the command doesn't show in --help for users who haven't opted in to the diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index e922f2a3ba..a84e99f1c7 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -84,6 +84,10 @@ import { SqlFixTool } from "../altimate/tools/sql-fix" import { SqlAutocompleteTool } from "../altimate/tools/sql-autocomplete" import { SqlDiffTool } from "../altimate/tools/sql-diff" import { DataDiffTool } from "../altimate/tools/data-diff" +// altimate_change start - import dbt fault injection tool +import { DbtFaultInjectionTool } from "../altimate/tools/dbt-fault-injection" +import { isFaultInjectionEngineAvailable } from "../altimate/native/connections/fault-injection" +// altimate_change end import { FinopsQueryHistoryTool } from "../altimate/tools/finops-query-history" import { FinopsAnalyzeCreditsTool } from "../altimate/tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../altimate/tools/finops-expensive-queries" @@ -362,6 +366,9 @@ export namespace ToolRegistry { // altimate_change end const config = configInput?.config ?? (await Config.get()) // altimate_change end + // altimate_change start - only offer dbt_fault_injection when the engine provides FaultInjectionSession + const faultInjectionAvailable = await isFaultInjectionEngineAvailable() + // altimate_change end const question = ["app", "cli", "desktop"].includes(Flag.OPENCODE_CLIENT) || Flag.OPENCODE_ENABLE_QUESTION_TOOL // altimate_change start — v1.17.9: Tool.define returns an Effect; resolve the @@ -422,6 +429,9 @@ export namespace ToolRegistry { // altimate_change start — data-parity tool DataDiffTool, // altimate_change end + // altimate_change start - register dbt fault injection tool + ...(faultInjectionAvailable ? [DbtFaultInjectionTool] : []), + // altimate_change end FinopsQueryHistoryTool, FinopsAnalyzeCreditsTool, FinopsExpensiveQueriesTool, diff --git a/packages/opencode/test/altimate/carry-forward/tools-present.test.ts b/packages/opencode/test/altimate/carry-forward/tools-present.test.ts index 0c202876f3..f0255c9dfc 100644 --- a/packages/opencode/test/altimate/carry-forward/tools-present.test.ts +++ b/packages/opencode/test/altimate/carry-forward/tools-present.test.ts @@ -53,6 +53,7 @@ import { WarehouseRemoveTool } from "../../../src/altimate/tools/warehouse-remov import { WarehouseTestTool } from "../../../src/altimate/tools/warehouse-test" import { WarehouseDiscoverTool } from "../../../src/altimate/tools/warehouse-discover" import { DataDiffTool } from "../../../src/altimate/tools/data-diff" +import { DbtFaultInjectionTool } from "../../../src/altimate/tools/dbt-fault-injection" // altimate-core engine bridge tools (sampling of the 27) import { AltimateCoreCheckTool } from "../../../src/altimate/tools/altimate-core-check" @@ -103,6 +104,7 @@ describe("carry-forward: altimate tools present with stable ids", () => { expect(DbtProfilesTool.id).toBe("dbt_profiles") expect(DbtUnitTestGenTool.id).toBe("dbt_unit_test_gen") expect(DbtPrReviewTool.id).toBe("dbt_pr_review") + expect(DbtFaultInjectionTool.id).toBe("dbt_fault_injection") }) test("warehouse / connection tools keep their ids", () => { diff --git a/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts b/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts new file mode 100644 index 0000000000..d44edd5fd9 --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts @@ -0,0 +1,399 @@ +/** + * Real dbt check of every proposed test. + * + * The engine verifies a proposed test with SQL on the clean and the corrupted copy of the + * database. This test goes one step further: it takes every proposal from a real run, + * writes it into a scratch copy of the dbt project the way a user would (merged into the + * schema file that already describes the node, or saved as a singular test file), and runs + * real `dbt test` on the clean database and on the corrupted copy of the fault it was + * proposed for. It reports, as numbers: + * + * - proposals how many faults came back with a proposed test + * - accepted_by_dbt dbt parsed it and ran it (no compile/parse error) + * - passed_on_clean ... and it passed on the clean database + * - failed_on_corrupted ... and it failed on the corrupted copy + * + * Skipped unless FI_VERIFY_PROJECT names a prepared dbt-duckdb project directory (one with + * its built `.duckdb` file and a `profiles.yml`), and dbt, the duckdb driver and an engine + * with `FaultInjectionSession` are available (see fault-injection-e2e.test.ts). Nothing in + * the project directory is written: everything happens in copies under FI_VERIFY_WORK + * (default: a temporary directory), which are removed afterwards. + * + * source .work-fi/env.sh # scratch HOME: never run this against the real one + * FI_VERIFY_PROJECT=/path/to/project FI_VERIFY_OUT=/tmp/verify.json \ + * bun test test/altimate/fault-injection-dbt-verify.test.ts --timeout 14400000 + * + * FI_VERIFY_BUDGET faults to inject (default 1000, i.e. all candidates of a small project) + * FI_VERIFY_OUT write the per-proposal results as JSON here + */ + +import { describe, expect, test } from "bun:test" +import { spawnSync } from "child_process" +import fs from "fs" +import os from "os" +import path from "path" +import YAML from "yaml" +import { + loadFaultInjectionEngine, + runFaultInjection, + type LoadedEngine, +} from "../../src/altimate/native/connections/fault-injection" +import { proposedTestYaml } from "../../src/altimate/native/connections/fault-injection-report" + +const PROJECT = process.env.FI_VERIFY_PROJECT +const DBT = process.env.ALTIMATE_DBT_PATH ?? "dbt" + +async function engineAvailable(): Promise { + try { + await loadFaultInjectionEngine() + return true + } catch { + return false + } +} + +const READY = !!PROJECT && fs.existsSync(path.join(PROJECT ?? "", "dbt_project.yml")) && (await engineAvailable()) + +type Json = Record + +export interface ProposalCheck { + fault_id: string + form: string + test: string + accepted_by_dbt: boolean + passed_on_clean: boolean + failed_on_corrupted: boolean + clean_status?: string + corrupted_status?: string + note?: string +} + +export interface ProposalSummary { + proposals: number + accepted_by_dbt: number + passed_on_clean: number + failed_on_corrupted: number +} + +/** Merge a pasted `models:`/`sources:` block into an existing schema document, as a user would. */ +export function mergeBlock(existing: Json, block: Json): Json { + const byName = (list: Json[], name: string) => list.find((entry) => entry?.name === name) + const mergeEntry = (target: Json, item: Json) => { + for (const [key, value] of Object.entries(item)) { + if (key === "name") continue + if (key === "columns" || key === "tables") { + const list: Json[] = (target[key] ??= []) + for (const child of value as Json[]) { + const found = byName(list, child.name) + if (found) mergeEntry(found, child) + else list.push(child) + } + } else if (key === "data_tests" || key === "tests") { + // Use the key the entry already has: dbt rejects an entry that has both. + const existingKey = "data_tests" in target ? "data_tests" : "tests" in target ? "tests" : key + target[existingKey] = [...(target[existingKey] ?? []), ...(value as unknown[])] + } else if (!(key in target)) { + target[key] = value + } + } + } + for (const section of ["models", "seeds", "sources"]) { + for (const item of (block[section] ?? []) as Json[]) { + const list: Json[] = (existing[section] ??= []) + const found = byName(list, item.name) + if (found) mergeEntry(found, item) + else list.push(item) + } + } + return existing +} + +/** Give the proposal's test entry a tag, so `dbt test --select tag:...` runs only that test. */ +export function tagTestEntries(block: Json, tag: string): void { + const visit = (node: Json) => { + for (const key of ["data_tests", "tests"]) { + if (!Array.isArray(node[key])) continue + node[key] = node[key].map((entry: unknown) => { + const [name, args] = + typeof entry === "string" ? [entry, {}] : [Object.keys(entry as Json)[0], Object.values(entry as Json)[0] ?? {}] + const config = { ...((args as Json).config ?? {}), tags: [tag, "fi_proposed"] } + return { [name]: { ...(args as Json), config } } + }) + } + for (const child of [...(node.columns ?? []), ...(node.tables ?? [])]) visit(child) + } + for (const section of ["models", "seeds", "sources"]) for (const item of block[section] ?? []) visit(item) +} + +function copyProject(from: string, to: string) { + fs.cpSync(from, to, { + recursive: true, + // A symlink kept as a link would let a write into the copy reach the original. + dereference: true, + filter: (src) => !["target", "logs"].includes(path.basename(src)), + }) +} + +describe.skipIf(!READY)("every proposed test, run by real dbt", () => { + test( + "proposals are accepted by dbt, pass on clean data and fail on the corrupted copy", + async () => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + process.env.ALTIMATE_DBT_PATH = DBT + const root = fs.mkdtempSync(path.join(process.env.FI_VERIFY_WORK ?? os.tmpdir(), "fi-verify-")) + const userProject = path.join(root, "user-project") + const work = path.join(root, "work") + const mutants = path.join(root, "mutants") + fs.mkdirSync(work) + fs.mkdirSync(mutants) + copyProject(PROJECT!, userProject) + // The clean database is the one the project's own profile names. + const profiles = YAML.parse(fs.readFileSync(path.join(userProject, "profiles.yml"), "utf-8")) + const ownProject = YAML.parse(fs.readFileSync(path.join(userProject, "dbt_project.yml"), "utf-8")) + const ownProfile = profiles[ownProject.profile] + const ownPath = String(ownProfile.outputs[ownProfile.target].path) + const cleanDb = path.resolve(userProject, ownPath) + expect(fs.existsSync(cleanDb)).toBe(true) + expect(cleanDb.startsWith(userProject)).toBe(true) + + // 1. A real run. The sandbox copy of each fault is saved just before its snapshot + // queries: dbt has released the file by then, and it holds the corrupted producer + // and the rebuilt downstream models. + const real = await loadFaultInjectionEngine() + const savedFor = new Map() + const snapshotDir = (): string | undefined => { + const dir = fs.readdirSync(work).find((d) => d.startsWith("altimate-fault-injection-")) + return dir ? path.join(work, dir, "sandbox") : undefined + } + const engine: LoadedEngine = { + ...real, + Session: class { + inner: InstanceType + constructor(spec: string) { + this.inner = new real.Session(spec) + } + start() { + return this.inner.start() + } + report() { + return this.inner.report() + } + step(result: string) { + const next = this.inner.step(result) + const action = JSON.parse(next) + if (action.type === "ExecuteSql" && action.phase === "snapshot" && action.fault_id && !savedFor.has(action.fault_id)) { + const dir = snapshotDir() + const file = dir ? fs.readdirSync(dir).find((f) => f.endsWith(".duckdb")) : undefined + if (dir && file) { + // Same file name as the original: views store the database name. + const folder = path.join(mutants, `m${savedFor.size}`) + fs.mkdirSync(folder) + const copy = path.join(folder, file) + fs.copyFileSync(path.join(dir, file), copy) + savedFor.set(action.fault_id, copy) + } + } + return next + } + } as never, + } + const run = await runFaultInjection( + { project_dir: userProject, budget: Number(process.env.FI_VERIFY_BUDGET ?? 1000), work_dir: work }, + { loadEngine: async () => engine }, + ) + expect(run.success).toBe(true) + const report = run.report as Json + const nodes = (run.nodes ?? {}) as Record + const dbtVersion = run.dbt?.version + const slipped = (report.slipped_through ?? []) as Json[] + const withProposal = slipped.filter((r) => r.proposed_test) + + // 2. Write proposals into a scratch copy of the project, the way a user would. + const projectYml = YAML.parse(fs.readFileSync(path.join(userProject, "dbt_project.yml"), "utf-8")) + const profilesSource = YAML.parse(fs.readFileSync(path.join(userProject, "profiles.yml"), "utf-8")) + const profile = profilesSource[projectYml.profile] + const output = { ...profile.outputs[profile.target] } + output.path = "{{ env_var('FI_DB') }}" + output.threads = 1 + const profilesDir = path.join(root, "profiles") + fs.mkdirSync(profilesDir) + fs.writeFileSync( + path.join(profilesDir, "profiles.yml"), + YAML.stringify({ [projectYml.profile]: { target: profile.target, outputs: { [profile.target]: output } } }), + ) + + // Identical proposals (same node, column and test) are written once. + const dedupe = new Map() + const entries: Array<{ index: number; tag: string; result: Json }> = [] + withProposal.forEach((result, i) => { + const proposal = result.proposed_test as Json + const key = JSON.stringify([proposal.node_id, proposal.column, proposal.form, proposal.yaml, proposal.sql]) + const shared = dedupe.get(key) + if (shared === undefined) dedupe.set(key, i) + entries.push({ index: i, tag: `fi_prop_${shared ?? i}`, result }) + }) + const distinct = entries.filter((e) => dedupe.get(JSON.stringify([e.result.proposed_test.node_id, e.result.proposed_test.column, e.result.proposed_test.form, e.result.proposed_test.yaml, e.result.proposed_test.sql])) === e.index) + + const writeProposals = (dir: string, items: typeof entries) => { + const documents = new Map() + for (const { index, tag, result } of items) { + const proposal = result.proposed_test as Json + if (proposal.form === "singular_sql") { + const file = path.join(dir, proposal.file) + fs.mkdirSync(path.dirname(file), { recursive: true }) + fs.writeFileSync(file, `{{ config(tags=['${tag}', 'fi_proposed']) }}\n${proposal.sql}`) + continue + } + const block = YAML.parse(proposedTestYaml(proposal, nodes, dbtVersion)) as Json + tagTestEntries(block, tag) + const node = nodes[proposal.node_id] ?? {} + const relative = (proposal.resource_section === "sources" ? node.original_file_path : node.patch_path) as string | undefined + // A user pastes into this project's own YAML: a package's file is not theirs to edit, + // and dbt rejects a second entry for the package's resource. + const fromPackage = node.package_name && node.package_name !== report.project + if (!relative || fromPackage) { + fs.writeFileSync(path.join(dir, "models", `fi_proposed_${index}.yml`), YAML.stringify({ version: 2, ...block })) + continue + } + const file = path.join(dir, relative) + const doc = documents.get(file) ?? (fs.existsSync(file) ? YAML.parse(fs.readFileSync(file, "utf-8")) : { version: 2 }) + documents.set(file, mergeBlock(doc ?? { version: 2 }, block)) + } + for (const [file, doc] of documents) fs.writeFileSync(file, YAML.stringify(doc)) + } + + // 3. Real dbt. + const dbt = (dir: string, args: string[], db: string, targetDir: string) => { + const results = path.join(targetDir, "run_results.json") + fs.rmSync(results, { force: true }) + const proc = spawnSync( + DBT, + ["--no-use-colors", ...args, "--project-dir", dir, "--profiles-dir", profilesDir, "--target-path", targetDir, "--log-path", path.join(root, "dbt-logs")], + { encoding: "utf-8", env: { ...process.env, FI_DB: db, DBT_SEND_ANONYMOUS_USAGE_STATS: "false" }, maxBuffer: 1 << 28 }, + ) + const statuses = new Map() + if (fs.existsSync(results)) { + const manifest = JSON.parse(fs.readFileSync(path.join(targetDir, "manifest.json"), "utf-8")) + for (const r of JSON.parse(fs.readFileSync(results, "utf-8")).results) { + for (const tag of manifest.nodes?.[r.unique_id]?.tags ?? []) statuses.set(tag, r.status) + } + } + return { code: proc.status, statuses, parsed: fs.existsSync(results), output: `${proc.stdout}\n${proc.stderr}`.slice(-600) } + } + + // All proposals in one project first. If dbt cannot parse that (one bad proposal fails + // the whole parse), each is tried alone so one bad proposal does not hide the others. + const verify = path.join(root, "verify-project") + copyProject(userProject, verify) + writeProposals(verify, distinct) + const cleanStatuses = new Map() + const projectFor = new Map() + const parseNotes = new Map() + const combined = dbt(verify, ["test", "--select", "tag:fi_proposed"], cleanDb, path.join(root, "target-clean")) + if (combined.parsed) { + for (const [tag, status] of combined.statuses) cleanStatuses.set(tag, status) + for (const e of distinct) projectFor.set(e.tag, verify) + } else { + for (const e of distinct) { + const alone = path.join(root, `alone-${e.index}`) + copyProject(userProject, alone) + writeProposals(alone, [e]) + const one = dbt(alone, ["test", "--select", "tag:fi_proposed"], cleanDb, path.join(root, `target-alone-${e.index}`)) + if (one.parsed) { + const status = one.statuses.get(e.tag) + if (status) cleanStatuses.set(e.tag, status) + projectFor.set(e.tag, alone) + } else { + parseNotes.set(e.tag, one.output.slice(-300)) + } + } + } + + const checks: ProposalCheck[] = [] + for (const entry of entries) { + const proposal = entry.result.proposed_test as Json + const cleanStatus = cleanStatuses.get(entry.tag) + const mutant = savedFor.get(entry.result.fault_id) + const dir = projectFor.get(entry.tag) + let corruptedStatus: string | undefined + if (cleanStatus && mutant && dir) { + corruptedStatus = dbt(dir, ["test", "--select", `tag:${entry.tag}`], mutant, path.join(root, "target-mutant")).statuses.get(entry.tag) + } + checks.push({ + fault_id: entry.result.fault_id as string, + form: proposal.form ?? "schema_yaml", + test: proposal.test as string, + accepted_by_dbt: !!cleanStatus && !["error", "skipped"].includes(cleanStatus), + passed_on_clean: cleanStatus === "pass", + failed_on_corrupted: corruptedStatus === "fail", + clean_status: cleanStatus, + corrupted_status: corruptedStatus, + ...(parseNotes.has(entry.tag) ? { note: `dbt rejected it: ${parseNotes.get(entry.tag)}` } : {}), + }) + } + + const summary: ProposalSummary = { + proposals: checks.length, + accepted_by_dbt: checks.filter((c) => c.accepted_by_dbt).length, + passed_on_clean: checks.filter((c) => c.passed_on_clean).length, + failed_on_corrupted: checks.filter((c) => c.failed_on_corrupted).length, + } + const withheld = slipped.filter((r) => !r.proposed_test).length + const payload = { project: report.project, slipped_through: slipped.length, withheld_with_reason: withheld, unique_proposals: dedupe.size, combined_project_parsed: combined.parsed, ...summary, checks } + if (process.env.FI_VERIFY_OUT) fs.writeFileSync(process.env.FI_VERIFY_OUT, JSON.stringify(payload, null, 2)) + console.log( + `[dbt-verify] ${report.project}: ${summary.proposals} proposals (${dedupe.size} distinct); dbt accepted ${summary.accepted_by_dbt}, ` + + `passed on clean ${summary.passed_on_clean}, failed on corrupted ${summary.failed_on_corrupted}; ${withheld} slipped faults have no proposal (reason given)`, + ) + fs.rmSync(root, { recursive: true, force: true }) + + // The point of the check: nothing the report presents may be unusable, and there is something to check. + expect(summary.proposals).toBeGreaterThan(0) + expect(summary.accepted_by_dbt).toBe(summary.proposals) + expect(summary.passed_on_clean).toBe(summary.proposals) + expect(summary.failed_on_corrupted).toBe(summary.proposals) + }, + 14_400_000, + ) +}) + +describe("proposal YAML helpers", () => { + test("a pasted block is merged into the entry that already describes the node", () => { + const existing = { + version: 2, + models: [{ name: "orders", columns: [{ name: "id", data_tests: ["unique"] }] }], + } + const block = { + models: [{ name: "orders", columns: [{ name: "id", data_tests: ["not_null"] }, { name: "amount", data_tests: ["not_null"] }] }], + } + expect(mergeBlock(existing, block)).toEqual({ + version: 2, + models: [ + { + name: "orders", + columns: [ + { name: "id", data_tests: ["unique", "not_null"] }, + { name: "amount", data_tests: ["not_null"] }, + ], + }, + ], + }) + }) + + test("a merged test uses the key the entry already has", () => { + const existing = { models: [{ name: "orders", tests: ["a"] }] } + const block = { models: [{ name: "orders", data_tests: ["b"] }] } + expect(mergeBlock(existing, block)).toEqual({ models: [{ name: "orders", tests: ["a", "b"] }] }) + }) + + test("tags select exactly one proposal", () => { + const block: Json = { + sources: [{ name: "app", tables: [{ name: "t", columns: [{ name: "c", data_tests: ["not_null", { "dbt_utils.accepted_range": { min_value: 0 } }] }] }] }], + } + tagTestEntries(block, "fi_prop_3") + expect(block.sources[0].tables[0].columns[0].data_tests).toEqual([ + { not_null: { config: { tags: ["fi_prop_3", "fi_proposed"] } } }, + { "dbt_utils.accepted_range": { min_value: 0, config: { tags: ["fi_prop_3", "fi_proposed"] } } }, + ]) + }) +}) diff --git a/packages/opencode/test/altimate/fault-injection-driver.test.ts b/packages/opencode/test/altimate/fault-injection-driver.test.ts new file mode 100644 index 0000000000..0c07383e12 --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-driver.test.ts @@ -0,0 +1,1279 @@ +/** + * Fault-injection driver against a scripted fake session. + * + * No database, no dbt and no engine: the session replays a fixed list of + * actions and records what it is stepped with, and the sandbox and dbt runner + * are fakes. This pins the caller obligations of the engine contract — + * sequencing, error reporting and timeout handling. + */ + +import { describe, expect, test } from "bun:test" +import fs from "fs" +import os from "os" +import path from "path" +import { + CORE_DEV_PATH_ENV, + DuckDbSandbox, + FAULT_INJECTION_MIN_CORE_VERSION, + FaultInjectionInterrupted, + createDbtRunner, + driveFaultInjectionSession, + isFaultInjectionEngineAvailable, + resetFaultInjectionEngineAvailability, + loadFaultInjectionEngine, + performAction, + readDbtTarget, + renderValue, + resolveProducers, + resolveSandboxFactory, + runFaultInjection, + type DbtOutcome, + type FaultAction, + type FaultStepResult, + type PerformDeps, + type SqlTarget, +} from "../../src/altimate/native/connections/fault-injection" +import { + formatFaultInjection, + formatRate, + proposedTestYaml, + summarizeFaultInjection, +} from "../../src/altimate/native/connections/fault-injection-report" +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import { FaultInjectionCommand } from "../../src/cli/cmd/fault-injection" + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** Replays `actions` in order and records every result it is stepped with. */ +function scriptedSession(actions: FaultAction[]) { + const stepped: FaultStepResult[] = [] + let cursor = 0 + return { + stepped, + start: () => JSON.stringify(actions[cursor++]), + step: (resultJson: string) => { + stepped.push(JSON.parse(resultJson)) + return JSON.stringify(actions[cursor++]) + }, + report: () => JSON.stringify({ summary: { selected: 1 } }), + } +} + +class CatalogError extends Error {} + +interface FakeOptions { + /** SQL text -> rows, or an Error to throw. Unlisted SQL returns no rows. */ + sql?: Record + /** Outcomes returned by successive dbt invocations. */ + dbt?: Array + prepareError?: Error + releaseError?: Error +} + +function fakes(options: FakeOptions = {}) { + const log: string[] = [] + const dbtQueue = [...(options.dbt ?? [])] + const deps: PerformDeps = { + sandbox: { + async execute(target: SqlTarget, sql: string) { + log.push(`sql:${target}:${sql}`) + const scripted = options.sql?.[sql] + if (scripted instanceof Error) throw scripted + return scripted ?? [] + }, + isRelationMissing: (e: unknown) => e instanceof CatalogError, + async prepareSandbox() { + log.push("prepare") + if (options.prepareError) throw options.prepareError + }, + async release(target: SqlTarget) { + log.push(`release:${target}`) + if (options.releaseError) throw options.releaseError + }, + }, + dbt: { + async run(target: SqlTarget, args: string[]) { + log.push(`dbt:${target}:${args.join(" ")}`) + const next = dbtQueue.shift() + if (next instanceof Error) throw next + return next ?? { exitCode: 0, results: [], tail: "", timedOut: false } + }, + }, + } + return { deps, log } +} + +const DONE: FaultAction = { type: "Done", report: { summary: { killed: 1 } } } + +const sqlAction = (sequential: boolean, tasks: Array<[string, string, SqlTarget?, string?]>): FaultAction => ({ + type: "ExecuteSql", + id: "a1", + phase: "mutate", + sequential, + tasks: tasks.map(([id, sql, target, shape]) => ({ + id, + sql, + target: target ?? "Sandbox", + expected_shape: (shape ?? "RowSet") as "RowSet", + })), +}) + +const rebuild: FaultAction = { + type: "RebuildNodes", + id: "a3", + producer_id: "seed.p.raw", + fault_id: "p|seed.p.raw|drop_rows|*", + node_ids: ["model.p.stg"], + select: ["raw+"], + exclude: ["raw"], + full_refresh: true, +} + +const runTests: FaultAction = { + type: "RunTests", + id: "a4", + producer_id: "seed.p.raw", + fault_id: "p|seed.p.raw|drop_rows|*", + test_ids: ["test.p.unique_stg_id"], + select: ["raw+"], +} + +// --------------------------------------------------------------------------- +// Value rendering +// --------------------------------------------------------------------------- + +describe("renderValue", () => { + test("keeps every digit of a HUGEINT checksum", () => { + const checksum = 170141183460469231731687303715884105727n + expect(renderValue(checksum)).toBe("170141183460469231731687303715884105727") + // The same value through Number would be 1.7014118346046923e+38. + expect(renderValue(checksum)).not.toBe(String(Number(checksum))) + }) + + test("maps NULL to null and passes text through", () => { + expect(renderValue(null)).toBeNull() + expect(renderValue(undefined)).toBeNull() + expect(renderValue("2024-01-02")).toBe("2024-01-02") + expect(renderValue(42)).toBe("42") + expect(renderValue(false)).toBe("false") + }) +}) + +// --------------------------------------------------------------------------- +// Sequencing +// --------------------------------------------------------------------------- + +describe("driveFaultInjectionSession", () => { + test("performs each action in order and echoes its id", async () => { + const session = scriptedSession([ + sqlAction(false, [["t1", "SELECT COUNT(*), SUM(h) FROM x", "Baseline", "SingleRow"]]), + { type: "PrepareSandbox", id: "a2", purpose: "fault", producer_id: "seed.p.raw", fault_id: "p|seed.p.raw|drop_rows|*" }, + rebuild, + runTests, + DONE, + ]) + const { deps, log } = fakes({ + sql: { "SELECT COUNT(*), SUM(h) FROM x": [[99n, 941742165569565516398n]] }, + dbt: [ + { exitCode: 0, results: [{ unique_id: "model.p.stg", status: "success" }], tail: "", timedOut: false }, + { exitCode: 1, results: [{ unique_id: "test.p.unique_stg_id", status: "fail" }], tail: "", timedOut: false }, + ], + }) + + const outcome = await driveFaultInjectionSession(session, deps) + + expect(outcome.report).toEqual({ summary: { killed: 1 } }) + expect(outcome.actions).toEqual({ ExecuteSql: 1, PrepareSandbox: 1, RebuildNodes: 1, RunTests: 1 }) + expect(Object.keys(outcome.perFaultMs)).toEqual(["p|seed.p.raw|drop_rows|*"]) + expect(session.stepped).toEqual([ + { type: "Sql", id: "a1", responses: [{ id: "t1", rows: [["99", "941742165569565516398"]] }] }, + { type: "Ok", id: "a2" }, + { type: "NodeResults", id: "a3", results: [{ unique_id: "model.p.stg", status: "success" }] }, + { type: "NodeResults", id: "a4", results: [{ unique_id: "test.p.unique_stg_id", status: "fail" }] }, + ]) + // The sandbox is released before each dbt invocation, and dbt runs single-threaded. + expect(log).toEqual([ + "sql:Baseline:SELECT COUNT(*), SUM(h) FROM x", + "prepare", + "release:Sandbox", + "dbt:Sandbox:run --threads 1 --full-refresh --select raw+ --exclude raw", + "release:Sandbox", + "dbt:Sandbox:test --threads 1 --select raw+", + ]) + }) + + test("returns the engine's error without performing anything further", async () => { + const session = scriptedSession([{ type: "Error", message: "manifest has no nodes" }]) + const { deps, log } = fakes() + const outcome = await driveFaultInjectionSession(session, deps) + expect(outcome.error).toBe("manifest has no nodes") + expect(outcome.report).toBeUndefined() + expect(log).toEqual([]) + }) + + test("rejects an action type it does not know", async () => { + const session = scriptedSession([{ type: "Teleport", id: "a1" } as unknown as FaultAction]) + const outcome = await driveFaultInjectionSession(session, fakes().deps) + expect(outcome.error).toBe("Unexpected action type: Teleport") + }) + + test("reports each fault once to the progress callback, retries included", async () => { + const prepare: FaultAction = { type: "PrepareSandbox", id: "a1", purpose: "fault", producer_id: "seed.p.raw", fault_id: "f1" } + const control: FaultAction = { type: "PrepareSandbox", id: "a0", purpose: "control", producer_id: "seed.p.raw" } + const session = scriptedSession([control, prepare, { ...prepare, id: "a2" }, DONE]) + const events: unknown[] = [] + await driveFaultInjectionSession(session, fakes().deps, { onProgress: (e) => events.push(e) }) + expect(events).toEqual([ + { kind: "control", producer_id: "seed.p.raw" }, + { kind: "fault", fault_id: "f1", index: 1, total: 1 }, + ]) + }) + + test("stops with FaultInjectionInterrupted when the signal aborts mid-run", async () => { + const controller = new AbortController() + const session = scriptedSession([sqlAction(false, [["t1", "SELECT 1"]]), rebuild, DONE]) + const { deps, log } = fakes() + const original = deps.sandbox.execute + deps.sandbox.execute = async (target, sql) => { + controller.abort() + return original(target, sql) + } + deps.signal = controller.signal + + await expect(driveFaultInjectionSession(session, deps, { signal: controller.signal })).rejects.toBeInstanceOf( + FaultInjectionInterrupted, + ) + // The rebuild that followed the abort never started. + expect(log.some((entry) => entry.startsWith("dbt:"))).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// SQL batches +// --------------------------------------------------------------------------- + +describe("ExecuteSql", () => { + test("a sequential batch stops at the first error and omits later tasks", async () => { + const { deps, log } = fakes({ sql: { "DROP VIEW v": new Error("Binder Error: cannot drop") } }) + const result = await performAction( + sqlAction(true, [ + ["t1", "SET threads = 1", "Sandbox", "Statement"], + ["t2", "DROP VIEW v", "Sandbox", "Statement"], + ["t3", "CREATE TABLE v AS SELECT 1", "Sandbox", "Statement"], + ]) as any, + deps, + ) + expect(result).toEqual({ + type: "Sql", + id: "a1", + responses: [ + { id: "t1", rows: [] }, + { id: "t2", error: "Binder Error: cannot drop" }, + ], + }) + expect(log).toEqual(["sql:Sandbox:SET threads = 1", "sql:Sandbox:DROP VIEW v"]) + }) + + test("a non-sequential batch runs every task and reports errors per task", async () => { + const { deps } = fakes({ + sql: { "SELECT * FROM gone": new CatalogError("Catalog Error: Table with name gone does not exist!"), "SELECT 2": [[2]] }, + }) + const result = await performAction( + sqlAction(false, [ + ["t1", "SELECT * FROM gone"], + ["t2", "SELECT 2"], + ]) as any, + deps, + ) + expect(result).toEqual({ + type: "Sql", + id: "a1", + responses: [ + { id: "t1", error: "Catalog Error: Table with name gone does not exist!", relation_missing: true }, + { id: "t2", rows: [["2"]] }, + ], + }) + }) + + test("relation_missing is set only for catalog errors", async () => { + const { deps } = fakes({ sql: { "SELECT bad": new Error("Conversion Error: could not cast") } }) + const result = (await performAction(sqlAction(false, [["t1", "SELECT bad"]]) as any, deps)) as any + expect(result.responses[0].error).toContain("Conversion Error") + expect(result.responses[0]).not.toHaveProperty("relation_missing") + }) + + test("a statement returns no rows even when the driver reports some", async () => { + const { deps } = fakes({ sql: { "CREATE TABLE t AS SELECT 1": [[1n]] } }) + const result = (await performAction( + sqlAction(true, [["t1", "CREATE TABLE t AS SELECT 1", "Sandbox", "Statement"]]) as any, + deps, + )) as any + expect(result.responses).toEqual([{ id: "t1", rows: [] }]) + }) + + test("error text is truncated", async () => { + const { deps } = fakes({ sql: { "SELECT 1": new Error("x".repeat(5000)) } }) + const result = (await performAction(sqlAction(false, [["t1", "SELECT 1"]]) as any, deps)) as any + expect(result.responses[0].error.length).toBe(500) + }) +}) + +// --------------------------------------------------------------------------- +// Sandbox and dbt failures +// --------------------------------------------------------------------------- + +describe("sandbox and dbt actions", () => { + test("a sandbox that cannot be prepared is reported as Failed", async () => { + const { deps } = fakes({ prepareError: new Error("ENOSPC: no space left on device") }) + const result = await performAction( + { type: "PrepareSandbox", id: "a2", purpose: "control", producer_id: "seed.p.raw" }, + deps, + ) + expect(result).toEqual({ type: "Failed", id: "a2", message: "ENOSPC: no space left on device" }) + }) + + test("a rebuild that writes no results is Failed, even on exit 0", async () => { + for (const exitCode of [0, 2]) { + const { deps } = fakes({ dbt: [{ exitCode, results: null, tail: "Runtime Error: database is locked", timedOut: false }] }) + const result = (await performAction(rebuild as any, deps)) as any + expect(result.type).toBe("Failed") + expect(result.id).toBe("a3") + expect(result.message).toContain("dbt run produced no results") + expect(result.message).toContain("database is locked") + expect(result).not.toHaveProperty("timed_out") + } + }) + + test("a dbt timeout is Failed with timed_out", async () => { + const { deps } = fakes({ dbt: [{ exitCode: null, results: null, tail: "dbt run timed out after 50ms", timedOut: true }] }) + expect(await performAction(rebuild as any, deps)).toEqual({ + type: "Failed", + id: "a3", + message: "dbt run timed out after 50ms", + timed_out: true, + }) + }) + + test("a failing model is a node result, not a failed action", async () => { + const { deps } = fakes({ + dbt: [{ exitCode: 1, results: [{ unique_id: "model.p.stg", status: "error" }], tail: "", timedOut: false }], + }) + expect(await performAction(rebuild as any, deps)).toEqual({ + type: "NodeResults", + id: "a3", + results: [{ unique_id: "model.p.stg", status: "error" }], + }) + }) + + test("dbt that cannot be started is Failed", async () => { + const { deps } = fakes({ dbt: [new Error("spawn dbt ENOENT")] }) + expect(await performAction(runTests as any, deps)).toEqual({ type: "Failed", id: "a4", message: "spawn dbt ENOENT" }) + }) + + test("dbt is not started when the sandbox cannot be released", async () => { + const { deps, log } = fakes({ releaseError: new Error("DETACH failed") }) + const result = (await performAction(rebuild as any, deps)) as any + expect(result.type).toBe("Failed") + expect(result.message).toContain("could not release the sandbox") + expect(log).toEqual(["release:Sandbox"]) + }) + + test("an empty selector never reaches dbt, which would rebuild everything", async () => { + const { deps, log } = fakes() + const result = await performAction({ ...(rebuild as any), select: [], exclude: [] }, deps) + expect(result).toEqual({ type: "NodeResults", id: "a3", results: [] }) + expect(log).toEqual([]) + }) + + test("no test results is fine only when no test was expected", async () => { + const none = { exitCode: 0, results: null, tail: "Nothing to do", timedOut: false } + const expectedSome = (await performAction(runTests as any, fakes({ dbt: [none] }).deps)) as any + expect(expectedSome.type).toBe("Failed") + const expectedNone = await performAction({ ...(runTests as any), test_ids: [] }, fakes({ dbt: [none] }).deps) + expect(expectedNone).toEqual({ type: "NodeResults", id: "a4", results: [] }) + }) + + test("an interrupt during dbt propagates instead of being reported as a failure", async () => { + const { deps } = fakes({ dbt: [new FaultInjectionInterrupted()] }) + await expect(performAction(rebuild as any, deps)).rejects.toBeInstanceOf(FaultInjectionInterrupted) + }) +}) + +// --------------------------------------------------------------------------- +// Early refusals +// --------------------------------------------------------------------------- + +function tempProject(profile: string): { project: string; work: string; cleanup: () => void } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-driver-test-")) + const project = path.join(root, "project") + const work = path.join(root, "work") + fs.mkdirSync(project) + fs.mkdirSync(work) + fs.writeFileSync(path.join(project, "dbt_project.yml"), 'name: "p"\nprofile: "p"\n') + fs.writeFileSync(path.join(project, "profiles.yml"), profile) + return { project, work, cleanup: () => fs.rmSync(root, { recursive: true, force: true }) } +} + +const neverLoaded = async (): Promise => { + throw new Error("the engine must not be loaded for an unsupported warehouse") +} + +describe("early refusals", () => { + test("an unsupported warehouse is named, with what is supported", () => { + expect(() => resolveSandboxFactory("snowflake")).toThrow( + "Fault injection does not support snowflake yet. It currently works on duckdb projects only", + ) + expect(() => resolveSandboxFactory("DuckDB")).not.toThrow() + }) + + test("a non-DuckDB project fails before the engine loads or anything is copied", async () => { + const { project, work, cleanup } = tempProject( + "p:\n target: dev\n outputs:\n dev:\n type: snowflake\n account: x\n", + ) + try { + const result = await runFaultInjection({ project_dir: project, work_dir: work }, { loadEngine: neverLoaded }) + expect(result.success).toBe(false) + expect(result.error).toContain("does not support snowflake yet") + expect(result.work_dir).toBeUndefined() + expect(fs.readdirSync(work)).toEqual([]) + } finally { + cleanup() + } + }) + + test("a directory that is not a dbt project is refused", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "fi-driver-test-")) + try { + const result = await runFaultInjection({ project_dir: dir }) + expect(result.success).toBe(false) + expect(result.error).toContain("No dbt_project.yml") + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + + const engineStub = async () => ({ Session: class {} as any, source: "package" as const }) + const dbtStub = async () => ({ path: "dbt", version: "1.10.0", env: {} }) + + const refusals: Array<[string, string, string]> = [ + ["an in-memory database", "path: ':memory:'", "not a local DuckDB file"], + ["MotherDuck", "path: 'md:prod'", "not a local DuckDB file"], + ["attached databases", "path: db.duckdb\n attach:\n - path: other.duckdb", 'sets "attach"'], + ["a database file that does not exist", "path: missing.duckdb", "DuckDB file not found"], + ] + for (const [what, config, message] of refusals) { + test(`refuses ${what} and leaves no work directory behind`, async () => { + const { project, work, cleanup } = tempProject( + `p:\n target: dev\n outputs:\n dev:\n type: duckdb\n ${config}\n`, + ) + try { + const result = await runFaultInjection( + { project_dir: project, work_dir: work }, + { loadEngine: engineStub, resolveDbt: dbtStub }, + ) + expect(result.success).toBe(false) + expect(result.error).toContain(message) + expect(result.work_dir_removed).toBe(true) + expect(fs.readdirSync(work)).toEqual([]) + } finally { + cleanup() + } + }) + } + + test("refuses a database with a pending write-ahead log", async () => { + const { project, work, cleanup } = tempProject("p:\n target: dev\n outputs:\n dev:\n type: duckdb\n path: db.duckdb\n") + try { + fs.writeFileSync(path.join(project, "db.duckdb"), "not really a database") + fs.writeFileSync(path.join(project, "db.duckdb.wal"), "") + const result = await runFaultInjection( + { project_dir: project, work_dir: work }, + { loadEngine: engineStub, resolveDbt: dbtStub }, + ) + expect(result.success).toBe(false) + expect(result.error).toContain("db.duckdb.wal exists") + expect(fs.readFileSync(path.join(project, "db.duckdb"), "utf-8")).toBe("not really a database") + expect(fs.readdirSync(work)).toEqual([]) + } finally { + cleanup() + } + }) +}) + +// --------------------------------------------------------------------------- +// Engine loading +// --------------------------------------------------------------------------- + +describe("isFaultInjectionEngineAvailable", () => { + const ok = async () => ({ Session: class {} as never, source: "package" as const }) + const lacking = async () => { + throw new Error(`Fault injection needs @altimateai/altimate-core ${FAULT_INJECTION_MIN_CORE_VERSION} or newer`) + } + + test("true when the engine loads (tool is registered)", async () => { + expect(await isFaultInjectionEngineAvailable(ok)).toBe(true) + }) + + test("false when the engine lacks the class (tool is not registered), without throwing", async () => { + expect(await isFaultInjectionEngineAvailable(lacking)).toBe(false) + }) + + test("the command path still reports the clear version error when the engine lacks the class", async () => { + const { project, work, cleanup } = tempProject( + "p:\n target: dev\n outputs:\n dev:\n type: duckdb\n path: x.duckdb\n", + ) + try { + const result = await runFaultInjection({ project_dir: project, work_dir: work }, { loadEngine: lacking }) + expect(result.success).toBe(false) + expect(result.error).toContain(`needs @altimateai/altimate-core ${FAULT_INJECTION_MIN_CORE_VERSION} or newer`) + } finally { + cleanup() + } + }) + + test("the default check is cached", async () => { + resetFaultInjectionEngineAvailability() + const first = isFaultInjectionEngineAvailable() + expect(isFaultInjectionEngineAvailable()).toBe(first) + await first + resetFaultInjectionEngineAvailability() + }) +}) + +describe("loadFaultInjectionEngine", () => { + class FakeSession {} + const withClass = async () => ({ FaultInjectionSession: FakeSession }) + const withoutClass = async () => ({ DataParitySession: class {} }) + + test("uses the package when it exports the class", async () => { + const engine = await loadFaultInjectionEngine({}, true, withClass) + expect(engine.source).toBe("package") + expect(engine.Session as unknown).toBe(FakeSession) + }) + + test("without the class, names the minimum version", async () => { + await expect(loadFaultInjectionEngine({}, true, withoutClass)).rejects.toThrow( + `Fault injection needs @altimateai/altimate-core ${FAULT_INJECTION_MIN_CORE_VERSION} or newer`, + ) + }) + + test("a package that cannot be imported is reported, not thrown raw", async () => { + const broken = async () => { + throw new Error("Cannot find module") + } + await expect(loadFaultInjectionEngine({}, true, broken)).rejects.toThrow( + "altimate-core NAPI module unavailable: Cannot find module", + ) + }) + + test("a published release ignores the development override and says so", async () => { + const env = { [CORE_DEV_PATH_ENV]: "/nonexistent/altimate-core-node" } + expect((await loadFaultInjectionEngine(env, false, withClass)).source).toBe("package") + await expect(loadFaultInjectionEngine(env, false, withoutClass)).rejects.toThrow( + `${CORE_DEV_PATH_ENV} is set but a published release does not honour it.`, + ) + }) + + test("a development override that does not exist is reported, not ignored", async () => { + await expect( + loadFaultInjectionEngine({ [CORE_DEV_PATH_ENV]: "/nonexistent/altimate-core-node" }, true, withClass), + ).rejects.toThrow(`${CORE_DEV_PATH_ENV} points at "/nonexistent/altimate-core-node", which does not exist.`) + }) + + test("a development override is loaded from a directory or a file, and must export the class", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "fi-driver-test-")) + try { + fs.writeFileSync(path.join(dir, "index.js"), "module.exports = {}\n") + await expect(loadFaultInjectionEngine({ [CORE_DEV_PATH_ENV]: dir }, true)).rejects.toThrow( + "does not export FaultInjectionSession", + ) + const good = path.join(dir, "good.js") + fs.writeFileSync(good, "module.exports = { FaultInjectionSession: class {} }\n") + const engine = await loadFaultInjectionEngine({ [CORE_DEV_PATH_ENV]: good }, true, withoutClass) + expect(engine).toMatchObject({ source: "dev-override", path: good }) + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + + test("the pinned package either has the class or produces the version message", async () => { + const core: any = await import("@altimateai/altimate-core") + const load = loadFaultInjectionEngine({}) + if (typeof core.FaultInjectionSession === "function") expect((await load).source).toBe("package") + else await expect(load).rejects.toThrow(`${FAULT_INJECTION_MIN_CORE_VERSION} or newer`) + }) +}) + +// --------------------------------------------------------------------------- +// Project and profile resolution +// --------------------------------------------------------------------------- + +describe("readDbtTarget", () => { + const twoTargets = + "p:\n target: \"{{ env_var('FI_TARGET', 'dev') }}\"\n outputs:\n" + + " dev:\n type: duckdb\n path: \"{{ env_var('FI_DB') }}\"\n" + + " prod:\n type: snowflake\n account: x\n" + + test("resolves the default target and env_var() values, keeping the raw output", async () => { + const { project, cleanup } = tempProject(twoTargets) + try { + const target = await readDbtTarget(project, { env: { FI_DB: "/data/db.duckdb" } }) + expect(target).toMatchObject({ profileName: "p", targetName: "dev", adapterType: "duckdb" }) + expect(target.output.path).toBe("/data/db.duckdb") + expect(target.rawOutput.path).toBe("{{ env_var('FI_DB') }}") + expect(target.profilesFile).toBe(path.join(project, "profiles.yml")) + } finally { + cleanup() + } + }) + + test("--target wins over DBT_TARGET, which wins over the profile default", async () => { + const { project, cleanup } = tempProject(twoTargets) + try { + expect((await readDbtTarget(project, { env: { FI_TARGET: "prod" } })).adapterType).toBe("snowflake") + expect((await readDbtTarget(project, { env: { DBT_TARGET: "prod" } })).targetName).toBe("prod") + expect((await readDbtTarget(project, { target: "dev", env: { DBT_TARGET: "prod" } })).targetName).toBe("dev") + } finally { + cleanup() + } + }) + + test("an explicit profiles directory wins over DBT_PROFILES_DIR and the project", async () => { + const { project, work, cleanup } = tempProject(twoTargets) + try { + fs.writeFileSync(path.join(work, "profiles.yml"), "p:\n target: only\n outputs:\n only:\n type: duckdb\n path: x.duckdb\n") + expect((await readDbtTarget(project, { env: { DBT_PROFILES_DIR: work } })).targetName).toBe("only") + const other = path.join(work, "other") + fs.mkdirSync(other) + fs.writeFileSync(path.join(other, "profiles.yml"), "p:\n target: third\n outputs:\n third:\n type: duckdb\n path: y.duckdb\n") + expect((await readDbtTarget(project, { profilesDir: other, env: { DBT_PROFILES_DIR: work } })).targetName).toBe("third") + } finally { + cleanup() + } + }) + + test("a missing profile or target is named", async () => { + const { project, cleanup } = tempProject("other:\n target: dev\n outputs: {}\n") + try { + await expect(readDbtTarget(project, { env: {} })).rejects.toThrow('Profile "p" is not defined') + fs.writeFileSync(path.join(project, "profiles.yml"), twoTargets) + await expect(readDbtTarget(project, { target: "staging", env: {} })).rejects.toThrow( + 'Target "staging" is not defined for profile "p"', + ) + } finally { + cleanup() + } + }) +}) + +describe("resolveProducers", () => { + const manifest = { + nodes: { + "model.p.orders": { resource_type: "model", name: "orders" }, + "seed.p.raw_orders": { resource_type: "seed", name: "raw_orders" }, + "test.p.unique_orders_id": { resource_type: "test", name: "orders" }, + }, + sources: { "source.p.shop.orders": { resource_type: "source", name: "orders", source_name: "shop" } }, + } + + test("matches models, seeds and sources by name, and never tests", () => { + expect(resolveProducers(manifest, "orders")).toEqual(["model.p.orders", "source.p.shop.orders"]) + expect(resolveProducers(manifest, "raw_orders")).toEqual(["seed.p.raw_orders"]) + }) + + test("accepts a unique id or a qualified source name", () => { + expect(resolveProducers(manifest, "model.p.orders")).toEqual(["model.p.orders"]) + expect(resolveProducers(manifest, "shop.orders")).toEqual(["source.p.shop.orders"]) + expect(resolveProducers(manifest, "nope")).toEqual([]) + }) +}) + +// --------------------------------------------------------------------------- +// What the DuckDB strategy refuses after parsing +// --------------------------------------------------------------------------- + +describe("DuckDbSandbox.assertManifestIsolated", () => { + const sandbox = new DuckDbSandbox({ + projectDir: "/project", + workDir: "/work", + profileName: "p", + targetName: "dev", + output: { type: "duckdb", path: "shop.duckdb" }, + rawOutput: { type: "duckdb", path: "shop.duckdb" }, + }) + const model = (extra: Record = {}) => ({ + resource_type: "model", + name: "orders", + database: "shop", + config: { materialized: "table" }, + ...extra, + }) + + test("accepts a project that stays inside the copied database", () => { + const manifest = { + nodes: { + a: model({ config: { materialized: "table", "post-hook": [{ sql: "analyze {{ this }}" }] } }), + b: { resource_type: "operation", name: "on-run-start-0", raw_code: "create schema if not exists audit" }, + // A disabled model is never built, so what it would do does not matter. + c: model({ database: "other", config: { enabled: false, materialized: "external" } }), + }, + sources: { s: { resource_type: "source", name: "raw", database: "SHOP" } }, + } + expect(() => sandbox.assertManifestIsolated(manifest)).not.toThrow() + }) + + const refused: Array<[string, Record, string]> = [ + ["an external materialization", { nodes: { a: model({ config: { materialized: "external" } }) } }, '"external" materialization'], + ["a model in another database", { nodes: { a: model({ database: "other" }) } }, 'a database other than "shop" (orders in other)'], + ["a source in another database", { nodes: {}, sources: { s: { resource_type: "source", name: "raw", database: "lake" } } }, "raw in lake"], + [ + "a post-hook that exports", + { nodes: { a: model({ config: { materialized: "table", "post-hook": [{ sql: "COPY (select 1) TO 'exports/o.parquet'" }] } }) } }, + "hook(s) run ATTACH, COPY or EXPORT DATABASE", + ], + [ + "an on-run-start that attaches another database", + { nodes: { op: { resource_type: "operation", name: "on-run-start-0", raw_code: "attach '/data/other.duckdb' as other" } } }, + "files outside the database (on-run-start-0)", + ], + ] + for (const [what, manifest, message] of refused) { + test(`refuses ${what}`, () => { + expect(() => sandbox.assertManifestIsolated(manifest)).toThrow(message) + expect(() => sandbox.assertManifestIsolated(manifest)).toThrow("Refusing to run") + }) + } + + test("a catalog error is recognised wherever its line starts", () => { + expect(sandbox.isRelationMissing(new Error("Catalog Error: Table with name x does not exist!"))).toBe(true) + // The driver prefixes its own explanation when the text contains "locked". + expect( + sandbox.isRelationMissing(new Error('Database "x" is locked by another process.\nCatalog Error: Table with name blocked_users does not exist!')), + ).toBe(true) + expect(sandbox.isRelationMissing(new Error("Binder Error: column not found in Catalog Error: text"))).toBe(false) + }) + + test("a database named like a reserved DuckDB catalog is refused", () => { + expect( + () => + new DuckDbSandbox({ + projectDir: "/project", + workDir: "/work", + profileName: "p", + targetName: "dev", + output: { type: "duckdb", path: "memory.duckdb" }, + rawOutput: { type: "duckdb", path: "memory.duckdb" }, + }), + ).toThrow('a DuckDB database called "memory" collides') + }) +}) + +// --------------------------------------------------------------------------- +// The dbt runner, against a fake dbt executable +// --------------------------------------------------------------------------- + +describe.skipIf(process.platform === "win32")("createDbtRunner", () => { + /** A stand-in for dbt: a shell script whose behaviour is chosen by FAKE_DBT_MODE. */ + function fakeDbt(): { runner: ReturnType; root: string; argvFile: string; cleanup: () => void; mode: (m: string) => void } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-dbt-runner-")) + const script = path.join(root, "dbt") + const argvFile = path.join(root, "argv.txt") + fs.mkdirSync(path.join(root, "project")) + fs.writeFileSync( + script, + [ + "#!/bin/sh", + `printf '%s\\n' "$@" > "${argvFile}"`, + `env | grep '^DBT_' | sort > "${argvFile}.env"`, + `pwd > "${argvFile}.cwd"`, + 'target=""; prev=""', + 'for a in "$@"; do [ "$prev" = "--target-path" ] && target="$a"; prev="$a"; done', + 'case "$FAKE_DBT_MODE" in', + ' results) mkdir -p "$target"; echo \'{"results":[{"unique_id":"model.p.a","status":"success","extra":1}]}\' > "$target/run_results.json"; echo "Done. PASS=1" ;;', + ' noresults) echo "Runtime Error: could not open database" >&2; exit 2 ;;', + " hang) sleep 30 ;;", + "esac", + ].join("\n"), + { mode: 0o755 }, + ) + const env: Record = { ...process.env, DBT_DEFER: "true", DBT_FULL_REFRESH: "false", DBT_USER: "kept" } + const runner = createDbtRunner({ + dbtPath: script, + env, + projectDir: path.join(root, "project"), + workDir: root, + dbtProfile: "p", + dbtTarget: "dev", + timeoutMs: 400, + profilesDir: (target) => path.join(root, `profiles-${target.toLowerCase()}`), + }) + return { + runner, + root, + argvFile, + cleanup: () => fs.rmSync(root, { recursive: true, force: true }), + mode: (m) => void (env.FAKE_DBT_MODE = m), + } + } + + test("points dbt at the copy's profile, target and log paths and reads run_results.json", async () => { + const fake = fakeDbt() + try { + fake.mode("results") + const outcome = await fake.runner.run("Sandbox", ["run", "--select", "a+"]) + expect(outcome).toEqual({ + exitCode: 0, + results: [{ unique_id: "model.p.a", status: "success" }], + tail: "Done. PASS=1\n", + timedOut: false, + }) + expect(fs.readFileSync(fake.argvFile, "utf-8").trim().split("\n")).toEqual([ + "--no-use-colors", + "run", + "--project-dir", + path.join(fake.root, "project"), + "--profiles-dir", + path.join(fake.root, "profiles-sandbox"), + "--profile", + "p", + "--target", + "dev", + "--target-path", + path.join(fake.root, "target-sandbox"), + "--log-path", + path.join(fake.root, "logs"), + "--select", + "a+", + ]) + expect(fs.realpathSync(fs.readFileSync(`${fake.argvFile}.cwd`, "utf-8").trim())).toBe( + fs.realpathSync(path.join(fake.root, "project")), + ) + const dbtEnv = fs.readFileSync(`${fake.argvFile}.env`, "utf-8") + // Flags that would change what the rebuild does are not inherited; the project's own variables are. + expect(dbtEnv).not.toContain("DBT_DEFER=") + expect(dbtEnv).not.toContain("DBT_FULL_REFRESH=") + expect(dbtEnv).toContain("DBT_USER=kept") + expect(dbtEnv).toContain(`DBT_TARGET_PATH=${path.join(fake.root, "target-sandbox")}`) + expect(dbtEnv).toContain(`DBT_PROFILES_DIR=${path.join(fake.root, "profiles-sandbox")}`) + } finally { + fake.cleanup() + } + }) + + test("results left by an earlier invocation are never mistaken for this one's", async () => { + const fake = fakeDbt() + try { + fake.mode("results") + await fake.runner.run("Sandbox", ["run"]) + fake.mode("noresults") + const outcome = await fake.runner.run("Sandbox", ["run"]) + expect(outcome.exitCode).toBe(2) + expect(outcome.results).toBeNull() + expect(outcome.tail).toContain("could not open database") + } finally { + fake.cleanup() + } + }) + + test("a run that exceeds the timeout is stopped and reported as timed out", async () => { + const fake = fakeDbt() + try { + fake.mode("hang") + const started = Date.now() + const outcome = await fake.runner.run("Sandbox", ["run"]) + expect(outcome.timedOut).toBe(true) + expect(outcome.results).toBeNull() + expect(outcome.tail).toBe("dbt run timed out after 400ms") + expect(Date.now() - started).toBeLessThan(10_000) + } finally { + fake.cleanup() + } + }) + + test("an abort stops dbt and rejects with FaultInjectionInterrupted", async () => { + const fake = fakeDbt() + try { + fake.mode("hang") + const controller = new AbortController() + const pending = fake.runner.run("Baseline", ["build"], controller.signal) + // Abort once the fake dbt is demonstrably running. + const poll = setInterval(() => { + if (!fs.existsSync(`${fake.argvFile}.cwd`)) return + clearInterval(poll) + controller.abort() + }, 10) + await expect(pending).rejects.toBeInstanceOf(FaultInjectionInterrupted) + // Already aborted: dbt is not started at all. + fs.rmSync(fake.argvFile) + await expect(fake.runner.run("Baseline", ["build"], controller.signal)).rejects.toBeInstanceOf(FaultInjectionInterrupted) + expect(fs.existsSync(fake.argvFile)).toBe(false) + } finally { + fake.cleanup() + } + }) + + test("a dbt executable that does not exist rejects instead of hanging", async () => { + const runner = createDbtRunner({ + dbtPath: "/nonexistent/dbt", + env: {}, + projectDir: os.tmpdir(), + workDir: os.tmpdir(), + dbtProfile: "p", + dbtTarget: "dev", + profilesDir: () => os.tmpdir(), + }) + await expect(runner.run("Sandbox", ["run"])).rejects.toThrow("ENOENT") + }) +}) + +// --------------------------------------------------------------------------- +// The CLI command's exit codes +// --------------------------------------------------------------------------- + +describe("fault-injection command", () => { + const summary = (catchRate: number | null) => ({ + candidates: 4, selected: 4, executed: 4, killed: 1, slipped_through: 3, inert: 0, invalid: 0, skipped: 0, catch_rate: catchRate, + }) + const okResult = (catchRate: number | null) => ({ + success: true, + warehouse: "duckdb", + budget: 20, + report: { project: "p", dialect: "duckdb", dialect_verified: true, summary: summary(catchRate), slipped_through: [], results: [], controls: [], skipped: [], warnings: [] }, + }) + + /** Run the handler with the driver replaced, capturing what it prints and the exit code it sets. */ + async function run(result: Record, args: Record = {}) { + let received: Record | undefined + // Trigger lazy registration first so it cannot overwrite the stub. + try { + await Dispatcher.call("__trigger_hook__" as any, {} as any) + } catch {} + Dispatcher.register("dbt.fault_injection", async (params: any) => { + received = params + return result + }) + const stdout: string[] = [] + const stderr: string[] = [] + const write = process.stdout.write + const error = console.error + const before = process.exitCode + process.stdout.write = ((chunk: any) => (stdout.push(String(chunk)), true)) as typeof process.stdout.write + console.error = (...parts: unknown[]) => void stderr.push(parts.join(" ")) + try { + process.exitCode = 0 + await (FaultInjectionCommand.handler as (a: any) => Promise)({ budget: 20, format: "text", ...args }) + return { exitCode: process.exitCode, stdout: stdout.join(""), stderr: stderr.join("\n"), received } + } finally { + process.stdout.write = write + console.error = error + process.exitCode = before ?? 0 + // Handlers are process-wide; put the real one back for whatever runs next. + Dispatcher.register("dbt.fault_injection", (params: any) => runFaultInjection(params)) + } + } + + test("a successful run prints the report on stdout and exits 0", async () => { + const out = await run(okResult(0.25), { project: "/p", model: "orders", "profiles-dir": "/profiles", "work-dir": "/w" }) + expect(out.exitCode).toBe(0) + expect(out.stdout).toContain("Catch rate: 25.0%") + expect(out.received).toMatchObject({ project_dir: "/p", model: "orders", budget: 20, profiles_dir: "/profiles", work_dir: "/w" }) + expect(out.received!.signal).toBeInstanceOf(AbortSignal) + }) + + test("--format json prints the whole result", async () => { + const out = await run(okResult(0.25), { format: "json" }) + expect(JSON.parse(out.stdout).report.summary.killed).toBe(1) + }) + + test("--fail-under fails below the threshold and passes at it", async () => { + expect((await run(okResult(0.25), { "fail-under": 25 })).exitCode).toBe(0) + const below = await run(okResult(0.25), { "fail-under": 26 }) + expect(below.exitCode).toBe(1) + expect(below.stderr).toContain("Catch rate 25.0% is below --fail-under 26.") + }) + + test("--fail-under cannot pass when nothing was measured", async () => { + const out = await run(okResult(null), { "fail-under": 10 }) + expect(out.exitCode).toBe(1) + expect(out.stderr).toContain("There is no catch rate to compare with --fail-under 10.") + }) + + test("a failed run exits 1 with the reason on stderr; an interrupted one exits 130", async () => { + const failed = await run({ success: false, error: "The project does not build" }) + expect(failed.exitCode).toBe(1) + expect(failed.stdout).toBe("") + expect(failed.stderr).toContain("Fault injection failed: The project does not build") + expect((await run({ success: false, interrupted: true, error: "Interrupted" })).exitCode).toBe(130) + }) + + test("invalid options are rejected before anything runs", async () => { + for (const args of [{ budget: 0 }, { budget: 2.5 }, { "fail-under": 101 }, { seed: -1 }, { seed: Number.NaN }]) { + const out = await run(okResult(0.5), args) + expect(out.exitCode).toBe(1) + expect(out.received).toBeUndefined() + } + }) +}) + +// --------------------------------------------------------------------------- +// Report rendering +// --------------------------------------------------------------------------- + +describe("report rendering", () => { + const nodes = { + "seed.p.raw_orders": { name: "raw_orders", resource_type: "seed", patch_path: "models/schema.yml" }, + "model.p.orders": { name: "orders", resource_type: "model" }, + "source.p.shop.payments": { name: "payments", resource_type: "source", source_name: "shop" }, + } + const slipped = { + fault_id: "p|seed.p.raw_orders|unit_scale|amount", + producer_id: "seed.p.raw_orders", + template: "unit_scale", + column: "amount", + outcome: "survived_impactful", + affected_rows: 5, + producer_rows: 99, + tests_run: 4, + failed_tests: [], + build_errors: [], + changed_relations: [ + { + unique_id: "model.p.orders", + baseline_rows: 99, + rows: 99, + comparison: { method: "keyed", key_columns: ["order_id"], rows_added: 0, rows_removed: 0, rows_changed: 5, columns: [{ column: "amount", rows_changed: 5 }] }, + }, + ], + proposed_test: { + test: "dbt_utils.accepted_range", + node_id: "seed.p.raw_orders", + resource_section: "seeds", + column: "amount", + yaml: "- dbt_utils.accepted_range:\n min_value: 0\n max_value: 30", + rationale: "`amount` lies between 0 and 30 in the baseline.", + verification: { baseline_failures: 0, sandbox_failures: 5, catches_fault: true }, + }, + } + const result = { + success: true, + warehouse: "duckdb", + database: "/p/db.duckdb", + original_unchanged: true, + work_dir: "/tmp/altimate-fault-injection-x", + work_dir_removed: true, + budget: 2, + dbt: { path: "dbt", version: "1.10.23" }, + nodes, + report: { + project: "p", + dialect: "duckdb", + dialect_verified: true, + summary: { candidates: 9, selected: 2, executed: 2, killed: 1, slipped_through: 1, inert: 0, invalid: 0, skipped: 0, catch_rate: 0.5 }, + slipped_through: [slipped], + results: [slipped], + controls: [], + skipped: [], + warnings: [], + }, + timing: { total_ms: 30_000, setup_ms: 5_000, run_ms: 25_000, per_fault_ms: { a: 6_000, b: 4_000 } }, + } + + test("leads with the catch rate and lists each slipped fault with its test", () => { + const text = formatFaultInjection(result as any) + expect(text).toContain("Catch rate: 50.0% (1 of 2 faults that mattered were caught)") + expect(text).toContain("2 of 9 candidate faults were selected (budget 2)") + expect(text).toContain("1. seed raw_orders: `amount` multiplied by 100 in 5 of 99 rows") + expect(text).toContain("model orders: 5 rows changed (amount: 5) of 99 (matched on order_id)") + expect(text).toContain("Verified on the data: passes on the clean data and fails on the corrupted copy (5 failing).") + expect(text).toContain("not evidence that the data in the warehouse today is wrong") + expect(text).toContain("4 tests ran and none failed because of the fault.") + expect(text).toContain("/p/db.duckdb is unchanged (same size and modification time as before the run).") + expect(text).toContain("The work directory /tmp/altimate-fault-injection-x has been removed.") + expect(text).toContain("2 faults 10s (5.0s each on average)") + expect(summarizeFaultInjection(result as any)).toBe("catch rate 50.0% (1 caught, 1 slipped through)") + }) + + test("the proposed test is a complete, correctly indented schema block", () => { + expect(proposedTestYaml(slipped.proposed_test, nodes, "1.10.23")).toBe( + [ + "seeds:", + " - name: raw_orders", + " columns:", + " - name: amount", + " data_tests:", + " - dbt_utils.accepted_range:", + " min_value: 0", + " max_value: 30", + ].join("\n"), + ) + }) + + test("a relation-level test on a source, on a dbt that predates data_tests", () => { + const proposal = { + test: "dbt_expectations.expect_table_row_count_to_be_between", + node_id: "source.p.shop.payments", + resource_section: "sources", + yaml: "- dbt_expectations.expect_table_row_count_to_be_between:\n min_value: 99", + } + expect(proposedTestYaml(proposal, nodes, "1.7.4")).toBe( + [ + "sources:", + " - name: shop", + " tables:", + " - name: payments", + " tests:", + " - dbt_expectations.expect_table_row_count_to_be_between:", + " min_value: 99", + ].join("\n"), + ) + }) + + test("a proposed test that was not verified is not presented as verified", () => { + const unverified = { ...slipped, proposed_test: { ...slipped.proposed_test, verification: undefined } } + const text = formatFaultInjection({ ...result, report: { ...result.report, slipped_through: [unverified] } } as any) + expect(text).toContain("Not verified against the data.") + expect(text).not.toContain("Verified on the data") + }) + + test("a fault with no stable test says why instead of proposing a brittle one", () => { + const withheld = { + ...slipped, + proposed_test: undefined, + proposal_note: "`days_open` is computed from the current date or time, so a range would fail as the clock moves.", + } + const text = formatFaultInjection({ ...result, report: { ...result.report, slipped_through: [withheld] } } as any) + expect(text).toContain( + "No test is proposed for this fault: `days_open` is computed from the current date or time, so a range would fail as the clock moves.", + ) + }) + + test("a singular test is shown as a file to add, with the reason it is not a schema entry", () => { + const singular = { + ...slipped, + proposed_test: { + form: "singular_sql", + test: "range (singular test)", + node_id: "model.pkg.stg_orders", + resource_section: "models", + column: "amount", + yaml: "", + file: "tests/fault_injection/fault_injection__stg_orders__range__amount.sql", + sql: "SELECT * FROM {{ ref('pkg', 'stg_orders') }} WHERE \"amount\" < 0 OR \"amount\" > 1000\n", + rationale: "`amount` lies between 0 and 30 in the baseline.", + verification: { baseline_failures: 0, sandbox_failures: 5, catches_fault: true }, + }, + } + const packaged = { ...nodes, "model.pkg.stg_orders": { name: "stg_orders", resource_type: "model", package_name: "pkg" } } + const text = formatFaultInjection({ ...result, nodes: packaged, report: { ...result.report, slipped_through: [singular] } } as any) + expect(text).toContain("belongs to the installed package pkg") + expect(text).toContain("This is a singular test instead (plain SQL, no package needed).") + expect(text).toContain("Save as tests/fault_injection/fault_injection__stg_orders__range__amount.sql") + expect(text).toContain("SELECT * FROM {{ ref('pkg', 'stg_orders') }}") + expect(text).not.toContain("data_tests:") + }) + + test("the reason a singular test was chosen is the engine's, not a guess about packages", () => { + const quoted = { + ...slipped, + proposed_test: { + form: "singular_sql", + test: "not_null (singular test)", + node_id: "seed.p.raw_orders", + resource_section: "seeds", + column: "Order ID", + yaml: "", + file: "tests/fault_injection/fault_injection__raw_orders__not_null__order_id.sql", + sql: 'SELECT * FROM {{ ref(\'raw_orders\') }} WHERE "Order ID" IS NULL\n', + singular_reason: + "dbt writes column names and accepted values into its standard tests without quoting them, so a name or value like this one would break the standard test.", + verification: { baseline_failures: 0, sandbox_failures: 5, catches_fault: true }, + }, + } + const text = formatFaultInjection({ ...result, report: { ...result.report, slipped_through: [quoted] } } as any) + expect(text).toContain("without quoting them") + expect(text).not.toContain("has not installed") + expect(text).not.toContain("installed package") + }) + + test("row-level detail from a sampled comparison says what the per-column counts cover", () => { + const sampled = { + ...slipped, + changed_relations: [ + { + unique_id: "model.p.orders", + baseline_rows: 600000, + rows: 600000, + comparison: { method: "keyed", key_columns: ["order_id"], rows_added: 0, rows_removed: 0, rows_changed: 30000, sampled_rows: 20000, columns: [{ column: "amount", rows_changed: 20000 }] }, + }, + ], + } + const text = formatFaultInjection({ ...result, report: { ...result.report, slipped_through: [sampled] } } as any) + expect(text).toContain("30000 rows changed (amount: 20000; the per-column counts cover the first 20000 changed rows by key)") + }) + + test("a failure says what happened and what became of the copies", () => { + const text = formatFaultInjection({ + success: false, + error: "The project does not build", + database: "/p/db.duckdb", + original_unchanged: true, + work_dir: "/tmp/x", + work_dir_removed: true, + }) + expect(text).toBe( + "Fault injection failed: The project does not build\n" + + "/p/db.duckdb is unchanged (same size and modification time as before the run).\n" + + "The work directory /tmp/x has been removed.", + ) + expect(summarizeFaultInjection({ success: false, interrupted: true })).toBe("interrupted") + }) + + test("never claims the database is unchanged when that was not established", () => { + const changed = formatFaultInjection({ + success: false, + error: "/p/db.duckdb changed during the run (its size or modification time differs).", + database: "/p/db.duckdb", + original_unchanged: false, + work_dir: "/tmp/x", + work_dir_removed: false, + }) + expect(changed).not.toContain("is unchanged") + expect(changed).toContain("The work directory /tmp/x could not be removed; delete it by hand.") + // Refused before the database was examined: nothing is claimed either way. + const early = formatFaultInjection({ success: false, error: "Refusing to run", database: "/p/db.duckdb", work_dir: "/tmp/x", work_dir_removed: true }) + expect(early).not.toContain("is unchanged") + }) + + test("the catch rate is rounded down, so a slipped fault never reads as 100%", () => { + expect(formatRate(0.9996)).toBe("99.9%") + expect(formatRate(0.6538461538461539)).toBe("65.3%") + expect(formatRate(1)).toBe("100.0%") + expect(formatRate(0)).toBe("0.0%") + }) + + test("the budget is blamed only when it was the limit", () => { + const summary = { ...result.report.summary, selected: 2, candidates: 9 } + const limited = formatFaultInjection({ ...result, budget: 2, report: { ...result.report, summary } } as any) + expect(limited).toContain("2 of 9 candidate faults were selected (budget 2). Raise the budget to run more.") + const notLimited = formatFaultInjection({ ...result, budget: 20, report: { ...result.report, summary } } as any) + expect(notLimited).toContain("2 of 9 candidate faults were selected.") + expect(notLimited).not.toContain("Raise the budget") + }) + + test("a node from an installed package is not presented as editable in this project", () => { + const packaged = { ...nodes, "seed.p.raw_orders": { ...nodes["seed.p.raw_orders"], package_name: "vendor_pkg" } } + const text = formatFaultInjection({ ...result, nodes: packaged } as any) + expect(text).toContain("The node belongs to the installed package vendor_pkg (models/schema.yml there).") + expect(text).not.toContain("(the node is described in models/schema.yml)") + // The same node in the project itself is pointed at directly. + const own = { ...nodes, "seed.p.raw_orders": { ...nodes["seed.p.raw_orders"], package_name: "p" } } + expect(formatFaultInjection({ ...result, nodes: own } as any)).toContain("(the node is described in models/schema.yml)") + }) + + test("names that are not plain identifiers are quoted in the YAML", () => { + const proposal = { test: "not_null", node_id: "seed.p.raw_orders", resource_section: "seeds", column: "Order Date", yaml: "- not_null" } + expect(proposedTestYaml(proposal, nodes, "1.10.0")).toContain(' - name: "Order Date"\n') + }) +}) diff --git a/packages/opencode/test/altimate/fault-injection-duckdb.test.ts b/packages/opencode/test/altimate/fault-injection-duckdb.test.ts new file mode 100644 index 0000000000..e797a6b5da --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-duckdb.test.ts @@ -0,0 +1,234 @@ +/** + * The DuckDB sandbox strategy against real DuckDB files. No engine and no dbt. + * + * What matters here is the property the whole driver rests on: after + * `release()`, ANOTHER PROCESS can open the sandbox file read-write, because + * that is what dbt does next. Closing a node-duckdb handle does not give that + * guarantee (the lock survives until garbage collection), which is why the + * strategy ATTACHes and DETACHes instead of opening the file. + * + * Opt-in like the other real-DuckDB suites: ALTIMATE_DUCKDB_E2E=1. A missing + * driver is then a failure, not a skip. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { connect } from "../../../drivers/src/duckdb" +import { DuckDbSandbox } from "../../src/altimate/native/connections/fault-injection" + +const RUN = process.env["ALTIMATE_DUCKDB_E2E"] === "1" +const ddbTest = RUN ? test : test.skip + +const DRIVER_PATH = fileURLToPath(new URL("../../../drivers/src/duckdb.ts", import.meta.url)) + +/** Open `file` read-write from a separate OS process, create a table, and exit. */ +function writeFromAnotherProcess(file: string, scratch: string): { ok: boolean; stderr: string } { + const script = path.join(scratch, "other-process.ts") + fs.writeFileSync( + script, + [ + `const { connect } = await import(${JSON.stringify(DRIVER_PATH)})`, + `const c = await connect({ type: "duckdb", path: process.argv[2] })`, + `await c.connect()`, + `await c.execute("CREATE OR REPLACE TABLE main.written_elsewhere AS SELECT 42 AS answer")`, + `process.exit(0)`, + ].join("\n"), + ) + const child = Bun.spawnSync([process.execPath, script, file], { stdout: "pipe", stderr: "pipe" }) + return { ok: child.exitCode === 0, stderr: child.stderr.toString() } +} + +describe("DuckDbSandbox on real DuckDB files", () => { + let root: string + let project: string + let work: string + let original: string + let sandbox: DuckDbSandbox + + beforeEach(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-duckdb-")) + project = path.join(root, "project") + work = path.join(root, "work") + fs.mkdirSync(project) + fs.mkdirSync(work) + original = path.join(project, "shop.duckdb") + if (!RUN) return + // Create the "user's" database without this process ever holding its lock afterwards. + const boot = await connect({ type: "duckdb", path: ":memory:" }) + await boot.connect() + await boot.execute(`ATTACH '${original}' AS shop`) + await boot.execute("CREATE TABLE shop.main.raw_orders AS SELECT * FROM (VALUES (1, 10), (2, 20), (3, 30)) t(id, amount)") + await boot.execute("CREATE VIEW shop.main.stg_orders AS SELECT id AS order_id, amount FROM shop.main.raw_orders") + await boot.execute("DETACH shop") + await boot.close() + const output = { type: "duckdb", path: "shop.duckdb", schema: "main", threads: 8, settings: { memory_limit: "1GB" } } + sandbox = new DuckDbSandbox({ projectDir: project, workDir: work, profileName: "shop", targetName: "dev", output, rawOutput: output }) + await sandbox.setup() + }) + + afterEach(async () => { + await sandbox?.close() + fs.rmSync(root, { recursive: true, force: true }) + }) + + ddbTest("setup copies the database under its own name and points both profiles at the copies", async () => { + const YAML = (await import("yaml")).default + for (const [target, dir] of [["Baseline", "baseline"], ["Sandbox", "sandbox"]] as const) { + const profile = YAML.parse(fs.readFileSync(path.join(sandbox.profilesDir(target), "profiles.yml"), "utf-8")) + expect(profile.shop.target).toBe("dev") + expect(profile.shop.outputs.dev).toEqual({ + type: "duckdb", + path: path.join(work, dir, "shop.duckdb"), + schema: "main", + threads: 1, + settings: { memory_limit: "1GB", threads: 1 }, + }) + } + expect(fs.existsSync(path.join(work, "baseline", "shop.duckdb"))).toBe(true) + expect(sandbox.original).toBe(original) + expect(await sandbox.verifyOriginalUntouched()).toEqual({ status: "unchanged" }) + }) + + ddbTest("temp_directory in settings and config_options is redirected into the work directory", async () => { + const YAML = (await import("yaml")).default + const elsewhere = path.join(root, "user-temp") + const output = { + type: "duckdb", + path: "shop.duckdb", + schema: "main", + settings: { memory_limit: "1GB", temp_directory: elsewhere }, + config_options: { temp_directory: elsewhere, preserve_insertion_order: false }, + } + const other = new DuckDbSandbox({ projectDir: project, workDir: path.join(root, "work2"), profileName: "shop", targetName: "dev", output, rawOutput: output }) + try { + fs.mkdirSync(path.join(root, "work2")) + await other.setup() + for (const target of ["Baseline", "Sandbox"] as const) { + const profile = YAML.parse(fs.readFileSync(path.join(other.profilesDir(target), "profiles.yml"), "utf-8")) + const out = profile.shop.outputs.dev + const expected = path.join(root, "work2", "duckdb-temp", target.toLowerCase()) + expect(out.settings.temp_directory).toBe(expected) + expect(out.config_options.temp_directory).toBe(expected) + expect(out.settings.memory_limit).toBe("1GB") + expect(out.config_options.preserve_insertion_order).toBe(false) + } + expect(fs.existsSync(elsewhere)).toBe(false) + } finally { + await other.close() + } + }) + + ddbTest("the baseline is read-only and reports relations with their kinds", async () => { + expect(await sandbox.execute("Baseline", 'SELECT COUNT(*), SUM(amount) FROM "main"."stg_orders"')).toEqual([[3n, 60n]]) + await expect(sandbox.execute("Baseline", 'CREATE TABLE "main"."x" AS SELECT 1')).rejects.toThrow("read-only") + expect(await sandbox.listRelations()).toEqual([ + { + database: "shop", + schema: "main", + name: "raw_orders", + relation_type: "table", + columns: [ + { name: "id", data_type: "INTEGER" }, + { name: "amount", data_type: "INTEGER" }, + ], + }, + { + database: "shop", + schema: "main", + name: "stg_orders", + relation_type: "view", + columns: [ + { name: "order_id", data_type: "INTEGER" }, + { name: "amount", data_type: "INTEGER" }, + ], + }, + ]) + }) + + ddbTest("only a catalog error counts as a missing relation", async () => { + const missing = await sandbox.execute("Baseline", 'SELECT * FROM "main"."nope"').catch((e) => e) + expect(sandbox.isRelationMissing(missing)).toBe(true) + const binder = await sandbox.execute("Baseline", 'SELECT nope FROM "main"."raw_orders"').catch((e) => e) + expect(binder).toBeInstanceOf(Error) + expect(sandbox.isRelationMissing(binder)).toBe(false) + }) + + ddbTest("after release another process can write the sandbox, and its writes are seen on re-attach", async () => { + await sandbox.prepareSandbox() + const file = path.join(work, "sandbox", "shop.duckdb") + await sandbox.execute("Sandbox", 'CREATE TABLE "main"."mutated" AS SELECT * FROM "main"."raw_orders" WHERE id <> 2') + + // While attached, this process holds the lock: the other process must fail. This is + // the situation dbt would be in without release(), and it makes the next assertion mean something. + const blocked = writeFromAnotherProcess(file, root) + expect(blocked.ok).toBe(false) + expect(blocked.stderr).toContain("lock") + + await sandbox.release("Sandbox") + expect(fs.existsSync(`${file}.wal`)).toBe(false) + const free = writeFromAnotherProcess(file, root) + expect(free.stderr).toBe("") + expect(free.ok).toBe(true) + + expect(await sandbox.execute("Sandbox", 'SELECT answer FROM "main"."written_elsewhere"')).toEqual([[42]]) + expect(await sandbox.execute("Sandbox", 'SELECT COUNT(*) FROM "main"."mutated"')).toEqual([[2n]]) + // Nothing reached the baseline or the original. + await expect(sandbox.execute("Baseline", 'SELECT * FROM "main"."mutated"')).rejects.toThrow("Catalog Error") + expect(await sandbox.verifyOriginalUntouched()).toEqual({ status: "unchanged" }) + }) + + ddbTest("prepareSandbox discards the previous sandbox, including while it is attached", async () => { + await sandbox.prepareSandbox() + await sandbox.execute("Sandbox", 'DROP VIEW "main"."stg_orders"') + await sandbox.prepareSandbox() + expect(await sandbox.execute("Sandbox", 'SELECT COUNT(*) FROM "main"."stg_orders"')).toEqual([[3n]]) + }) + + ddbTest("refuses to attach a sandbox path that resolves to the original database", async () => { + await sandbox.prepareSandbox() + const file = path.join(work, "sandbox", "shop.duckdb") + fs.rmSync(file) + fs.symlinkSync(original, file) + await expect(sandbox.execute("Sandbox", 'DROP TABLE "main"."raw_orders"')).rejects.toThrow( + "Refusing to run: cannot establish that", + ) + fs.rmSync(file) + fs.linkSync(original, file) + // A hard link: caught by inode, or already by path where realpath follows the link (macOS). + await expect(sandbox.execute("Sandbox", 'DROP TABLE "main"."raw_orders"')).rejects.toThrow( + "Refusing to run: cannot establish that", + ) + expect(await sandbox.execute("Baseline", 'SELECT COUNT(*) FROM "main"."raw_orders"')).toEqual([[3n]]) + expect(await sandbox.verifyOriginalUntouched()).toEqual({ status: "unchanged" }) + }) + + ddbTest("a failed attach leaves no half-attached connection behind", async () => { + // No sandbox has been prepared yet: the file does not exist. + await expect(sandbox.execute("Sandbox", "SELECT 1")).rejects.toThrow() + await sandbox.prepareSandbox() + expect(await sandbox.execute("Sandbox", 'SELECT COUNT(*) FROM "main"."raw_orders"')).toEqual([[3n]]) + }) + + ddbTest("a change to the original database is noticed", async () => { + fs.appendFileSync(original, "x") + const check = await sandbox.verifyOriginalUntouched() + expect(check.status).toBe("changed") + }) + + ddbTest("a database with a pending write-ahead log is refused", async () => { + fs.writeFileSync(`${original}.wal`, "") + const output = { type: "duckdb", path: "shop.duckdb" } + const other = new DuckDbSandbox({ + projectDir: project, + workDir: fs.mkdtempSync(path.join(root, "work2-")), + profileName: "shop", + targetName: "dev", + output, + rawOutput: output, + }) + await expect(other.setup()).rejects.toThrow("shop.duckdb.wal exists") + }) +}) diff --git a/packages/opencode/test/altimate/fault-injection-e2e.test.ts b/packages/opencode/test/altimate/fault-injection-e2e.test.ts new file mode 100644 index 0000000000..f34c10538e --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-e2e.test.ts @@ -0,0 +1,279 @@ +/** + * Fault injection end to end: the real engine, real DuckDB, real dbt. + * + * Runs the driver on the dbt-duckdb project in `fixtures/fault-injection/project` + * (copied to a temp dir and built there first, so the fixture stays read-only). + * + * Skipped unless all of these are available: + * - dbt with the duckdb adapter: `ALTIMATE_DBT_PATH`, or `dbt` on PATH + * - the `duckdb` Node driver + * - `FaultInjectionSession`: an `@altimateai/altimate-core` that exports it, or, + * for development, `ALTIMATE_CORE_DEV_PATH` pointing at a locally built + * `crates/altimate-core-node` + * + * ALTIMATE_DBT_PATH=/path/to/venv/bin/dbt \ + * ALTIMATE_CORE_DEV_PATH=/path/to/altimate-core/crates/altimate-core-node \ + * bun test test/altimate/fault-injection-e2e.test.ts --timeout 600000 + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { execFileSync } from "child_process" +import { createHash } from "crypto" +import fs from "fs" +import os from "os" +import path from "path" +import { loadFaultInjectionEngine, runFaultInjection } from "../../src/altimate/native/connections/fault-injection" +import { formatFaultInjection } from "../../src/altimate/native/connections/fault-injection-report" +import type { DbtFaultInjectionResult } from "../../src/altimate/native/types" + +const FIXTURE = path.join(import.meta.dir, "fixtures", "fault-injection", "project") +const TIMEOUT_MS = 600_000 + +function findDbtWithDuckdb(): string | null { + const candidate = process.env.ALTIMATE_DBT_PATH ?? "dbt" + try { + const out = execFileSync(candidate, ["--version"], { encoding: "utf-8", timeout: 30_000, stdio: ["ignore", "pipe", "pipe"] }) + return /duckdb/i.test(out) ? candidate : null + } catch { + return null + } +} + +async function engineAvailable(): Promise { + try { + await loadFaultInjectionEngine() + return true + } catch { + return false + } +} + +async function duckdbDriverAvailable(): Promise { + try { + const { connect } = await import("@altimateai/drivers/duckdb") + const connector = await connect({ type: "duckdb", path: ":memory:" }) + await connector.connect() + await connector.close() + return true + } catch { + return false + } +} + +const DBT = findDbtWithDuckdb() +const READY = DBT !== null && (await engineAvailable()) && (await duckdbDriverAvailable()) + +const sha256 = (file: string) => createHash("sha256").update(fs.readFileSync(file)).digest("hex") + +describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () => { + let root: string + let project: string + let work: string + let database: string + let hashBefore: string + let savedDbtPath: string | undefined + + beforeAll(() => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + savedDbtPath = process.env.ALTIMATE_DBT_PATH + process.env.ALTIMATE_DBT_PATH = DBT! + root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-e2e-")) + project = path.join(root, "project") + work = path.join(root, "work") + fs.cpSync(FIXTURE, project, { recursive: true }) + fs.mkdirSync(work) + // Build the "user's" database. Target and logs go outside the project so the + // assertions below can tell whether the run itself wrote any. + execFileSync( + DBT!, + ["build", "--project-dir", project, "--profiles-dir", project, "--target-path", path.join(root, "setup-target"), "--log-path", path.join(root, "setup-logs")], + { cwd: project, stdio: "pipe", timeout: 120_000, env: { ...process.env, DBT_SEND_ANONYMOUS_USAGE_STATS: "false" } }, + ) + database = path.join(project, "fi_fixture.duckdb") + hashBefore = sha256(database) + }, TIMEOUT_MS) + + afterAll(() => { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + if (savedDbtPath === undefined) delete process.env.ALTIMATE_DBT_PATH + else process.env.ALTIMATE_DBT_PATH = savedDbtPath + if (root) fs.rmSync(root, { recursive: true, force: true }) + }) + + /** The user's files are exactly as they were and nothing of ours is left behind. */ + function expectNoTrace(result: DbtFaultInjectionResult) { + expect(sha256(database)).toBe(hashBefore) + expect(fs.readdirSync(project).sort()).toEqual( + ["README.md", "dbt_project.yml", "fi_fixture.duckdb", "models", "profiles.yml", "seeds"].sort(), + ) + expect(result.work_dir_removed).toBe(true) + expect(result.original_unchanged).toBe(true) + expect(fs.readdirSync(work)).toEqual([]) + } + + let full: DbtFaultInjectionResult + let budgeted: DbtFaultInjectionResult + + /** Drop what legitimately differs between two runs: timings and the scratch directory. */ + const comparable = (text: string) => + text + .split("\n") + .filter((line) => !line.startsWith("Took ")) + .join("\n") + .replace(/The work directory \S+/, "The work directory ") + + test( + "finds the faults the fixture's tests miss and the ones they catch", + async () => { + full = await runFaultInjection({ project_dir: project, model: "raw_orders", budget: 50, work_dir: work }) + expect(full.error).toBeUndefined() + expect(full.success).toBe(true) + + const report = full.report! + const summary = report.summary + expect(summary.executed).toBe(summary.selected) + expect(summary.executed).toBeGreaterThan(0) + expect(summary.killed + summary.slipped_through + summary.inert + summary.invalid).toBe(summary.executed) + expect(summary.catch_rate).toBeCloseTo(summary.killed / (summary.killed + summary.slipped_through), 10) + expect(report.dialect_verified).toBe(true) + // --model restricts the run to one producer. + expect(new Set(report.results.map((r: any) => r.producer_id))).toEqual(new Set(["seed.fi_fixture.raw_orders"])) + + const outcome = (suffix: string) => report.results.find((r: any) => r.fault_id.endsWith(suffix)) + // `id` carries unique + not_null, and customer_id a relationships test: these are caught. + expect(outcome("|duplicate_rows|*").outcome).toBe("killed") + expect(outcome("|null_out|id").outcome).toBe("killed") + expect(outcome("|orphan_fk|customer_id").outcome).toBe("killed") + expect(outcome("|duplicate_rows|*").failed_tests.length).toBeGreaterThan(0) + + // `amount` has no test at all: a 100x error flows into customer_orders unnoticed. + const scaled = outcome("|unit_scale|amount") + expect(scaled.outcome).toBe("survived_impactful") + expect(scaled.failed_tests).toEqual([]) + const changed = scaled.changed_relations.find((c: any) => c.unique_id === "model.fi_fixture.customer_orders") + expect(changed.comparison.method).toBe("keyed") + expect(changed.comparison.columns.map((c: any) => c.column)).toEqual(["total_amount"]) + expect(changed.comparison.rows_changed).toBeGreaterThan(0) + expect(scaled.proposed_test.verification).toEqual({ + baseline_failures: 0, + sandbox_failures: scaled.affected_rows, + catches_fault: true, + }) + expect(report.slipped_through.map((r: any) => r.fault_id)).toContain(scaled.fault_id) + + const text = formatFaultInjection(full) + expect(text).toContain("Fault injection: fi_fixture (duckdb)") + expect(text).toContain("seed raw_orders: `amount` multiplied by 100 in") + expect(text).toContain(" seeds:\n - name: raw_orders\n columns:\n - name: amount\n") + + expectNoTrace(full) + }, + TIMEOUT_MS, + ) + + test( + "is deterministic: the same budget and seed select and classify the same faults", + async () => { + const again = await runFaultInjection({ project_dir: project, model: "raw_orders", budget: 2, work_dir: work }) + const first = await runFaultInjection({ project_dir: project, model: "raw_orders", budget: 2, work_dir: work }) + expect(again.success && first.success).toBe(true) + expect(again.report!.summary.executed).toBe(2) + expect(again.report!.results).toEqual(first.report!.results) + // A budgeted run agrees with the full run on the faults they share. + for (const result of again.report!.results) { + expect(full.report!.results.find((r: any) => r.fault_id === result.fault_id)).toEqual(result) + } + expectNoTrace(again) + budgeted = again + }, + TIMEOUT_MS, + ) + + test( + "the dbt_fault_injection tool returns what the driver returns, after asking permission", + async () => { + const { initTool } = await import("./tool-fixture") + const { DbtFaultInjectionTool } = await import("../../src/altimate/tools/dbt-fault-injection") + const tool = await initTool(DbtFaultInjectionTool) + const asked: any[] = [] + const result = await tool.execute( + { project_dir: project, model: "raw_orders", budget: 2 }, + { + sessionID: "test", + messageID: "test", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async (request: any) => void asked.push(request), + }, + ) + expect(asked).toHaveLength(1) + expect(asked[0].permission).toBe("bash") + expect(asked[0].patterns).toEqual([`dbt build --project-dir ${project}`]) + expect(result.metadata.success).toBe(true) + expect(result.metadata.executed).toBe(2) + expect(result.metadata.killed).toBe(budgeted.report!.summary.killed) + expect(result.metadata.slipped_through).toBe(budgeted.report!.summary.slipped_through) + expect(result.title).toContain("catch rate") + expect(comparable(result.output)).toBe(comparable(formatFaultInjection(budgeted))) + expect(sha256(database)).toBe(hashBefore) + }, + TIMEOUT_MS, + ) + + test( + "a project that does not build is reported and cleaned up", + async () => { + const broken = path.join(project, "models", "broken.sql") + fs.writeFileSync(broken, "select no_such_column from {{ ref('raw_orders') }}\n") + try { + const result = await runFaultInjection({ project_dir: project, budget: 1, work_dir: work }) + expect(result.success).toBe(false) + expect(result.error).toContain("The project does not build on a clean copy") + expect(result.error).toContain("model.fi_fixture.broken (error)") + expect(result.report).toBeUndefined() + fs.rmSync(broken) + expectNoTrace(result) + } finally { + fs.rmSync(broken, { force: true }) + } + }, + TIMEOUT_MS, + ) + + test( + "an interrupt mid-run stops dbt and removes the copies", + async () => { + const controller = new AbortController() + let faults = 0 + const result = await runFaultInjection({ + project_dir: project, + budget: 50, + work_dir: work, + signal: controller.signal, + on_progress: (event) => { + // Abort once a fault is under way, i.e. while a sandbox exists and dbt is about to run. + if (event.kind === "fault" && ++faults === 1) setTimeout(() => controller.abort(), 1_500) + }, + }) + expect(result.success).toBe(false) + expect(result.interrupted).toBe(true) + expect(result.report).toBeUndefined() + expectNoTrace(result) + }, + TIMEOUT_MS, + ) + + test( + "a dbt invocation that exceeds its timeout fails the run instead of hanging", + async () => { + const result = await runFaultInjection({ project_dir: project, budget: 1, work_dir: work, dbt_timeout_ms: 1 }) + expect(result.success).toBe(false) + expect(result.error).toContain("dbt parse timed out after 1ms") + expect(result.interrupted).toBeUndefined() + expectNoTrace(result) + }, + TIMEOUT_MS, + ) +}) diff --git a/packages/opencode/test/altimate/fault-injection-registration.test.ts b/packages/opencode/test/altimate/fault-injection-registration.test.ts new file mode 100644 index 0000000000..caeccb596f --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-registration.test.ts @@ -0,0 +1,84 @@ +/** + * The `dbt_fault_injection` agent tool is registered only when the installed engine + * provides `FaultInjectionSession`; the plain command keeps its own clear error + * (see fault-injection-driver.test.ts). + * + * The two cases use the real loader: pointing ALTIMATE_CORE_DEV_PATH at a directory that + * holds no engine makes the engine unavailable, and the availability cache is reset around + * each case so nothing leaks into other tests. + */ + +import { afterEach, describe, expect } from "bun:test" +import fs from "fs" +import os from "os" +import path from "path" +import { Effect } from "effect" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { ToolRegistry } from "@/tool/registry" +import { + CORE_DEV_PATH_ENV, + isFaultInjectionEngineAvailable, + resetFaultInjectionEngineAvailability, +} from "../../src/altimate/native/connections/fault-injection" +import { disposeAllInstances } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { TestConfig } from "../fixture/config" +import { Config } from "@/config/config" +import { Agent } from "@/agent/agent" +import { InstanceState } from "@/effect/instance-state" +import { RuntimeFlags } from "@/effect/runtime-flags" + +const configLayer = TestConfig.layer({ + directories: () => InstanceState.directory.pipe(Effect.map((dir) => [path.join(dir, ".opencode")])), +}) +const it = testEffect( + LayerNode.buildLayer(LayerNode.group([ToolRegistry.node, Agent.node]), { + replacements: [ + LayerNode.replace(Config.node, configLayer), + LayerNode.replace(RuntimeFlags.node, RuntimeFlags.layer()), + ], + }), +) + +const savedDevPath = process.env[CORE_DEV_PATH_ENV] +afterEach(async () => { + if (savedDevPath === undefined) delete process.env[CORE_DEV_PATH_ENV] + else process.env[CORE_DEV_PATH_ENV] = savedDevPath + resetFaultInjectionEngineAvailability() + await disposeAllInstances() +}) + +/** Run `body` with the engine made unavailable by pointing the dev override at an empty directory. */ +function withoutEngine(body: Effect.Effect): Effect.Effect { + return Effect.gen(function* () { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "fi-no-engine-")) + process.env[CORE_DEV_PATH_ENV] = empty + resetFaultInjectionEngineAvailability() + return yield* body + }) +} + +describe("dbt_fault_injection registration", () => { + it.instance("is not offered when the installed engine has no fault-injection class", () => + withoutEngine( + Effect.gen(function* () { + expect(yield* Effect.promise(() => isFaultInjectionEngineAvailable())).toBe(false) + const ids = yield* (yield* ToolRegistry.Service).ids() + expect(ids).not.toContain("dbt_fault_injection") + // The rest of the registry is unaffected. + expect(ids).toContain("sql_execute") + }), + ), + ) + + it.instance("is offered to the agent when the engine provides FaultInjectionSession", () => + Effect.gen(function* () { + // Needs a build of the engine that exports the class (ALTIMATE_CORE_DEV_PATH or a new enough package). + resetFaultInjectionEngineAvailability() + const available = yield* Effect.promise(() => isFaultInjectionEngineAvailable()) + const ids = yield* (yield* ToolRegistry.Service).ids() + expect(ids.includes("dbt_fault_injection")).toBe(available) + if (process.env[CORE_DEV_PATH_ENV]) expect(available).toBe(true) + }), + ) +}) diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/README.md b/packages/opencode/test/altimate/fixtures/fault-injection/project/README.md new file mode 100644 index 0000000000..e56417c8e6 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/README.md @@ -0,0 +1,9 @@ +# Fault-injection fixture + +A minimal dbt-duckdb project for `fault-injection-e2e.test.ts`: two seeds, a staging view and one +aggregate table. `status` and `amount` are deliberately untested, so faults in them slip through, +while the keys carry `unique`, `not_null` and `relationships` tests that catch duplicated rows, +NULL keys and orphaned foreign keys. + +The DuckDB file is not checked in. The test copies this directory to a temp dir and runs +`dbt build` there to create it. diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/dbt_project.yml b/packages/opencode/test/altimate/fixtures/fault-injection/project/dbt_project.yml new file mode 100644 index 0000000000..9231ef4a53 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/dbt_project.yml @@ -0,0 +1,18 @@ +name: "fi_fixture" +version: "1.0.0" +config-version: 2 +profile: "fi_fixture" + +model-paths: ["models"] +seed-paths: ["seeds"] + +models: + fi_fixture: + +materialized: table + +seeds: + fi_fixture: + raw_orders: + +column_types: + order_date: date + amount: integer diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/models/customer_orders.sql b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/customer_orders.sql new file mode 100644 index 0000000000..e8c04e1aaa --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/customer_orders.sql @@ -0,0 +1,9 @@ +select + c.id as customer_id, + c.name, + count(o.order_id) as order_count, + coalesce(sum(o.amount), 0) as total_amount, + min(o.order_date) as first_order +from {{ ref('raw_customers') }} c +left join {{ ref('stg_orders') }} o on o.customer_id = c.id +group by 1, 2 diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/models/schema.yml b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/schema.yml new file mode 100644 index 0000000000..e796b7bb17 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/schema.yml @@ -0,0 +1,35 @@ +version: 2 + +seeds: + - name: raw_customers + columns: + - name: id + data_tests: + - unique + - not_null + - name: raw_orders + columns: + - name: id + data_tests: + - unique + - not_null + +models: + - name: stg_orders + columns: + - name: order_id + data_tests: + - unique + - not_null + - name: customer_id + data_tests: + - not_null + - relationships: + to: ref('raw_customers') + field: id + - name: customer_orders + columns: + - name: customer_id + data_tests: + - unique + - not_null diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/models/stg_orders.sql b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/stg_orders.sql new file mode 100644 index 0000000000..9563e58211 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/models/stg_orders.sql @@ -0,0 +1,9 @@ +{{ config(materialized='view') }} + +select + id as order_id, + customer_id, + order_date, + status, + amount +from {{ ref('raw_orders') }} diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/profiles.yml b/packages/opencode/test/altimate/fixtures/fault-injection/project/profiles.yml new file mode 100644 index 0000000000..ce28d95be0 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/profiles.yml @@ -0,0 +1,8 @@ +fi_fixture: + target: dev + outputs: + dev: + type: duckdb + path: fi_fixture.duckdb + schema: main + threads: 1 diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_customers.csv b/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_customers.csv new file mode 100644 index 0000000000..a0abe02d0a --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_customers.csv @@ -0,0 +1,7 @@ +id,name,country +1,Ada,GB +2,Grace,US +3,Linus,FI +4,Margaret,US +5,Dennis,US +6,Barbara,US diff --git a/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_orders.csv b/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_orders.csv new file mode 100644 index 0000000000..c3b9ed8671 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/fault-injection/project/seeds/raw_orders.csv @@ -0,0 +1,25 @@ +id,customer_id,order_date,status,amount +1,1,2024-01-03,shipped,13 +2,2,2024-01-05,completed,16 +3,3,2024-01-07,returned,19 +4,4,2024-01-09,placed,22 +5,5,2024-01-11,shipped,25 +6,6,2024-01-13,completed,28 +7,1,2024-01-15,returned,31 +8,2,2024-01-17,placed,34 +9,3,2024-01-19,shipped,37 +10,4,2024-01-21,completed,40 +11,5,2024-01-23,returned,43 +12,6,2024-01-25,placed,46 +13,1,2024-01-27,shipped,49 +14,2,2024-01-29,completed,52 +15,3,2024-01-31,returned,55 +16,4,2024-02-02,placed,58 +17,5,2024-02-04,shipped,61 +18,6,2024-02-06,completed,64 +19,1,2024-02-08,returned,67 +20,2,2024-02-10,placed,70 +21,3,2024-02-12,shipped,73 +22,4,2024-02-14,completed,76 +23,5,2024-02-16,returned,79 +24,6,2024-02-18,placed,82 From b7e4d84b75d8ed5489004f162b721ecbe0fb5292 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 21:29:33 -0700 Subject: [PATCH 2/7] fix: [fault-injection] address independent review findings - exclude `unit_test` from the baseline build only on dbt-core 1.8 and newer, where the resource type exists - resolve a relative `project_dir` in the `dbt_fault_injection` tool against the session directory instead of the process working directory - treat only a missing table, view or schema as a missing relation - copy the content of symlinks into the project copy instead of the link Co-Authored-By: Claude Opus 5.5 (1M context) --- .../native/connections/fault-injection.ts | 18 ++++++++-- .../src/altimate/tools/dbt-fault-injection.ts | 3 +- .../altimate/fault-injection-driver.test.ts | 2 ++ .../test/altimate/fault-injection-e2e.test.ts | 33 +++++++++++-------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/packages/opencode/src/altimate/native/connections/fault-injection.ts b/packages/opencode/src/altimate/native/connections/fault-injection.ts index 6595d63304..4f02f2619f 100644 --- a/packages/opencode/src/altimate/native/connections/fault-injection.ts +++ b/packages/opencode/src/altimate/native/connections/fault-injection.ts @@ -604,7 +604,8 @@ export class DuckDbSandbox implements SandboxStrategy { isRelationMissing(error: unknown): boolean { // Anywhere at a line start: the driver may prefix DuckDB's text with its own explanation. - return /(^|\n)\s*(Error: )?Catalog Error:/i.test(errorText(error)) + // Only a missing relation: a missing function or type is also a Catalog Error but is a broken query. + return /(^|\n)\s*(Error: )?Catalog Error:\s*(Table|View|Schema)\b[^\n]*does not exist/i.test(errorText(error)) } async listRelations(): Promise { @@ -1204,6 +1205,14 @@ async function removeWorkDir(dir: string): Promise { return removed } +/** True for dbt-core 1.8 and newer, the first versions with a `unit_test` resource type. */ +export function supportsUnitTests(version: string): boolean { + const m = /(\d+)\.(\d+)/.exec(version) + if (!m) return true + const [major, minor] = [Number(m[1]), Number(m[2])] + return major > 1 || (major === 1 && minor >= 8) +} + /** Top-level entries of a project that dbt does not need and that must not be copied. */ const PROJECT_COPY_SKIP = new Set([".git", "target", "logs", "node_modules", "profiles.yml", ".user.yml"]) @@ -1212,10 +1221,12 @@ const PROJECT_COPY_SKIP = new Set([".git", "target", "logs", "node_modules", "pr * nothing it writes with a relative path (target/, logs/, a hook's export) * can land in the user's project. */ -async function copyProject(projectDir: string, dest: string, workDir: string): Promise { +export async function copyProject(projectDir: string, dest: string, workDir: string): Promise { const [root, work] = await Promise.all([fsp.realpath(projectDir), fsp.realpath(workDir)]) await fsp.cp(root, dest, { recursive: true, + // Copy what a symlink points at: a link kept as a link would lead dbt back to the user's files. + dereference: true, mode: fs.constants.COPYFILE_FICLONE, filter: (source) => { if (source === work || source.startsWith(work + path.sep)) return false @@ -1383,7 +1394,8 @@ export async function runFaultInjection( "--full-refresh", "--exclude", "resource_type:test", - "resource_type:unit_test", + // dbt rejects a resource type it does not know, and unit tests exist from 1.8. + ...(supportsUnitTests(dbt.version) ? ["resource_type:unit_test"] : []), ]) const baselineBuildMs = Date.now() - buildStarted if (built.results === null) { diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts index 1dafc629f3..5e91c6f712 100644 --- a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -1,6 +1,7 @@ import z from "zod" import path from "path" import { Tool } from "../../tool/tool" +import { Instance } from "../../project/instance" import { Dispatcher } from "../native" import { formatFaultInjection, summarizeFaultInjection } from "../native/connections/fault-injection-report" @@ -51,7 +52,7 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { profiles_dir: z.string().optional().describe("Directory containing profiles.yml. Defaults to dbt's lookup order."), }), async execute(args, ctx) { - const projectDir = path.resolve(args.project_dir ?? process.cwd()) + const projectDir = path.resolve(Instance.directory, args.project_dir ?? ".") // This runs dbt, which executes the project's own code. Ask as for any other command. const command = `dbt build --project-dir ${projectDir}` await ctx.ask({ diff --git a/packages/opencode/test/altimate/fault-injection-driver.test.ts b/packages/opencode/test/altimate/fault-injection-driver.test.ts index 0c07383e12..c75b38d895 100644 --- a/packages/opencode/test/altimate/fault-injection-driver.test.ts +++ b/packages/opencode/test/altimate/fault-injection-driver.test.ts @@ -14,6 +14,8 @@ import path from "path" import { CORE_DEV_PATH_ENV, DuckDbSandbox, + copyProject, + supportsUnitTests, FAULT_INJECTION_MIN_CORE_VERSION, FaultInjectionInterrupted, createDbtRunner, diff --git a/packages/opencode/test/altimate/fault-injection-e2e.test.ts b/packages/opencode/test/altimate/fault-injection-e2e.test.ts index f34c10538e..637f147865 100644 --- a/packages/opencode/test/altimate/fault-injection-e2e.test.ts +++ b/packages/opencode/test/altimate/fault-injection-e2e.test.ts @@ -121,6 +121,7 @@ describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () .filter((line) => !line.startsWith("Took ")) .join("\n") .replace(/The work directory \S+/, "The work directory ") + .replaceAll("/private/var/", "/var/") test( "finds the faults the fixture's tests miss and the ones they catch", @@ -196,21 +197,27 @@ describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () const { DbtFaultInjectionTool } = await import("../../src/altimate/tools/dbt-fault-injection") const tool = await initTool(DbtFaultInjectionTool) const asked: any[] = [] - const result = await tool.execute( - { project_dir: project, model: "raw_orders", budget: 2 }, - { - sessionID: "test", - messageID: "test", - agent: "test", - abort: new AbortController().signal, - messages: [], - metadata: () => {}, - ask: async (request: any) => void asked.push(request), - }, - ) + // A relative project_dir resolves against the session's directory, not the process cwd. + const { Instance } = await import("../../src/project/instance") + const result = await Instance.provide({ + directory: path.dirname(project), + fn: () => + tool.execute( + { project_dir: path.basename(project), model: "raw_orders", budget: 2 }, + { + sessionID: "test", + messageID: "test", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async (request: any) => void asked.push(request), + }, + ), + }) expect(asked).toHaveLength(1) expect(asked[0].permission).toBe("bash") - expect(asked[0].patterns).toEqual([`dbt build --project-dir ${project}`]) + expect(asked[0].patterns).toEqual([`dbt build --project-dir ${fs.realpathSync(project)}`]) expect(result.metadata.success).toBe(true) expect(result.metadata.executed).toBe(2) expect(result.metadata.killed).toBe(budgeted.report!.summary.killed) From 8f28515c956919257a194bf56734de4a046aefe7 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 21:57:03 -0700 Subject: [PATCH 3/7] fix: [fault-injection] address automated review feedback - ask for `external_directory` before copying a project or reading a profiles directory outside the workspace; ask for the dbt commands the run actually executes instead of a single `dbt build` proxy - resolve a relative `profiles_dir` against the session directory - fail when an explicit profiles directory has no `profiles.yml` instead of falling back to another profile - do not follow symlinks that lead into or around the scratch tree when copying the project - treat a dbt exit code above 1 as a dbt error, not as node results - compare `--fail-under` as `rate < failUnder / 100` to avoid rounding at the threshold - tolerate a keyed comparison without `key_columns`; quote `true`/`null`-like names in proposed YAML - docs: consistent timing example, `--work-dir` parent/child wording - tests: clean up scratch directories and restore environment on failure, bound the abort poll, add permission, profiles, symlink and rendering tests Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/docs/data-engineering/tools/dbt-tools.md | 9 +- .../connections/fault-injection-report.ts | 7 +- .../native/connections/fault-injection.ts | 33 +++++- .../src/altimate/tools/dbt-fault-injection.ts | 15 ++- .../opencode/src/cli/cmd/fault-injection.ts | 2 +- .../fault-injection-dbt-verify.test.ts | 19 +++- .../altimate/fault-injection-driver.test.ts | 47 +++++++- .../test/altimate/fault-injection-e2e.test.ts | 7 +- .../fault-injection-registration.test.ts | 3 +- .../fault-injection-tool-permission.test.ts | 104 ++++++++++++++++++ 10 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 packages/opencode/test/altimate/fault-injection-tool-permission.test.ts diff --git a/docs/docs/data-engineering/tools/dbt-tools.md b/docs/docs/data-engineering/tools/dbt-tools.md index 09e2e0ba9c..8d3484c9ff 100644 --- a/docs/docs/data-engineering/tools/dbt-tools.md +++ b/docs/docs/data-engineering/tools/dbt-tools.md @@ -251,16 +251,17 @@ that slipped through), the no-fault controls and per-fault timings. One `dbt build --full-refresh` of the models, seeds and snapshots on the copy, then for every corrupted relation three no-fault control runs, and for every fault one `dbt run` of the downstream -models plus one `dbt test`. All of it is single-threaded. The run above took 158 seconds on a -laptop for 52 faults over 6 relations. The controls are a fixed cost per relation, so a small budget +models plus one `dbt test`. All of it is single-threaded. The example above took 195 seconds on a +laptop for 50 faults. The controls are a fixed cost per relation, so a small budget spread over many relations is dominated by them: on a larger project, 20 faults spread over 17 relations took 186 seconds, 129 of them in profiling and controls. Use `--model` to concentrate the budget. ### Safety -dbt runs on a copy of the database and a copy of the project, both in a temporary directory (or -`--work-dir`), with its own profile, target path and log path. The directory is removed on success, +dbt runs on a copy of the database and a copy of the project, both in a temporary directory. `--work-dir` names the parent +of that directory, and the command creates an `altimate-fault-injection-*` directory inside it, +with its own profile, target path and log path. Only that directory is removed, on success, failure and interrupt, and the command reports where it was. The project's database is opened only to copy it, and the command checks afterwards that its size and modification time are unchanged. diff --git a/packages/opencode/src/altimate/native/connections/fault-injection-report.ts b/packages/opencode/src/altimate/native/connections/fault-injection-report.ts index 43d6a6a51d..04f173793f 100644 --- a/packages/opencode/src/altimate/native/connections/fault-injection-report.ts +++ b/packages/opencode/src/altimate/native/connections/fault-injection-report.ts @@ -66,7 +66,8 @@ function describeChange(changed: Json, nodes: Record 0) parts.push(`${plural(comparison.rows_added, "row")} added`) if (comparison.rows_removed > 0) parts.push(`${plural(comparison.rows_removed, "row")} removed`) const what = parts.length ? parts.join(", ") : "content changed" - return `${label}: ${what} of ${before} (matched on ${comparison.key_columns.join(", ")})` + const keys = (comparison.key_columns ?? []) as string[] + return `${label}: ${what} of ${before}${keys.length ? ` (matched on ${keys.join(", ")})` : ""}` } const why = comparison.note ? `; ${comparison.note}, so no row-level detail` : "" if (before === null || before === undefined || after === null || after === undefined) { @@ -94,7 +95,9 @@ function testsKey(dbtVersion: string | undefined): string { /** A name as a YAML scalar: plain when that is safe, quoted otherwise. */ function yamlScalar(value: string): string { - return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value) ? value : JSON.stringify(value) + // Plain scalars that YAML 1.1 reads as a boolean or null are quoted. + const reserved = /^(true|false|yes|no|on|off|y|n|null|~)$/i.test(value) + return !reserved && /^[A-Za-z_][A-Za-z0-9_]*$/.test(value) ? value : JSON.stringify(value) } /** The proposed test as a block that can be pasted into a schema file. */ diff --git a/packages/opencode/src/altimate/native/connections/fault-injection.ts b/packages/opencode/src/altimate/native/connections/fault-injection.ts index 4f02f2619f..1dbf45b4e4 100644 --- a/packages/opencode/src/altimate/native/connections/fault-injection.ts +++ b/packages/opencode/src/altimate/native/connections/fault-injection.ts @@ -950,9 +950,18 @@ async function runDbtAction( } return { type: "NodeResults", id, results: [] } } + // dbt exits 1 when nodes fail (their statuses are in the results) and 2 on an error of its own; + // partial results from the latter are not evidence about the nodes. + if (dbtCrashed(outcome)) { + return failed(id, `${what} stopped with an error (exit ${outcome.exitCode}): ${outcome.tail.slice(-ERROR_TEXT_LIMIT)}`) + } return { type: "NodeResults", id, results: outcome.results } } +function dbtCrashed(outcome: DbtOutcome): boolean { + return outcome.exitCode !== null && outcome.exitCode > 1 +} + /** Perform one engine action and return the result to step the session with. */ export async function performAction(action: PerformableAction, deps: PerformDeps): Promise { throwIfAborted(deps.signal) @@ -1107,9 +1116,16 @@ export async function readDbtTarget( projectDir, path.join(os.homedir(), ".dbt"), ].filter((d): d is string => Boolean(d)) - const profilesFile = candidates - .map((dir) => path.join(path.resolve(dir), "profiles.yml")) - .find((file) => fs.existsSync(file)) + // An explicit directory is never replaced by a default: a typo must not select another profile. + const explicit = options.profilesDir ? path.join(path.resolve(options.profilesDir), "profiles.yml") : undefined + if (explicit && !fs.existsSync(explicit)) { + throw new Error(`No profiles.yml in the requested profiles directory (${explicit}).`) + } + const profilesFile = + explicit ?? + candidates + .map((dir) => path.join(path.resolve(dir), "profiles.yml")) + .find((file) => fs.existsSync(file)) if (!profilesFile) { throw new Error(`No profiles.yml found (looked in ${candidates.join(", ")}).`) } @@ -1230,6 +1246,15 @@ export async function copyProject(projectDir: string, dest: string, workDir: str mode: fs.constants.COPYFILE_FICLONE, filter: (source) => { if (source === work || source.startsWith(work + path.sep)) return false + // A link is followed by the copy; refuse one that leads into, or around, the scratch tree. + try { + if (fs.lstatSync(source).isSymbolicLink()) { + const target = fs.realpathSync(source) + if (target === work || target.startsWith(work + path.sep) || work.startsWith(target + path.sep)) return false + } + } catch { + return false // dangling link + } const relative = path.relative(root, source) if (relative === "") return true if (!relative.includes(path.sep) && PROJECT_COPY_SKIP.has(relative)) return false @@ -1398,7 +1423,7 @@ export async function runFaultInjection( ...(supportsUnitTests(dbt.version) ? ["resource_type:unit_test"] : []), ]) const baselineBuildMs = Date.now() - buildStarted - if (built.results === null) { + if (built.results === null || dbtCrashed(built)) { throw new Error(describeDbtFailure("The project does not build: dbt build wrote no results", built)) } if (built.results.some((r) => ["error", "runtime error", "fail", "skipped"].includes(r.status))) { diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts index 5e91c6f712..c5c95fd7b0 100644 --- a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -2,6 +2,7 @@ import z from "zod" import path from "path" import { Tool } from "../../tool/tool" import { Instance } from "../../project/instance" +import { assertExternalDirectoryLegacy } from "../../tool/external-directory" import { Dispatcher } from "../native" import { formatFaultInjection, summarizeFaultInjection } from "../native/connections/fault-injection-report" @@ -54,11 +55,17 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { async execute(args, ctx) { const projectDir = path.resolve(Instance.directory, args.project_dir ?? ".") // This runs dbt, which executes the project's own code. Ask as for any other command. - const command = `dbt build --project-dir ${projectDir}` + const profilesDir = args.profiles_dir ? path.resolve(Instance.directory, args.profiles_dir) : undefined + // The project is copied and its dbt code is run, so a path outside the workspace needs the same + // external_directory permission the bash tool asks for. + await assertExternalDirectoryLegacy(ctx, projectDir, { kind: "directory" }) + if (profilesDir) await assertExternalDirectoryLegacy(ctx, profilesDir, { kind: "directory" }) + // The run executes these dbt commands on the copies; ask for each rather than for a proxy. + const commands = ["parse", "compile", "build", "run", "test"].map((c) => `dbt ${c} --project-dir ${projectDir}`) await ctx.ask({ permission: "bash", - patterns: [command], - always: [command], + patterns: commands, + always: commands, metadata: { project_dir: projectDir, budget: args.budget, model: args.model }, }) @@ -68,7 +75,7 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { model: args.model, budget: args.budget, target: args.target, - profiles_dir: args.profiles_dir, + profiles_dir: profilesDir, signal: ctx.abort, }) const summary = result.report?.summary diff --git a/packages/opencode/src/cli/cmd/fault-injection.ts b/packages/opencode/src/cli/cmd/fault-injection.ts index 638a24fabb..0657fe1ea0 100644 --- a/packages/opencode/src/cli/cmd/fault-injection.ts +++ b/packages/opencode/src/cli/cmd/fault-injection.ts @@ -139,7 +139,7 @@ export const FaultInjectionCommand = cmd({ // No fault was caught and none slipped through: nothing was measured, so the gate cannot pass. console.error(`There is no catch rate to compare with --fail-under ${failUnder}.`) process.exitCode = 1 - } else if (rate * 100 < failUnder) { + } else if (rate < failUnder / 100) { console.error(`Catch rate ${formatRate(rate)} is below --fail-under ${failUnder}.`) process.exitCode = 1 } diff --git a/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts b/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts index d44edd5fd9..b8ab3c9b7f 100644 --- a/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts +++ b/packages/opencode/test/altimate/fault-injection-dbt-verify.test.ts @@ -27,7 +27,7 @@ * FI_VERIFY_OUT write the per-proposal results as JSON here */ -import { describe, expect, test } from "bun:test" +import { afterEach, describe, expect, test } from "bun:test" import { spawnSync } from "child_process" import fs from "fs" import os from "os" @@ -135,12 +135,28 @@ function copyProject(from: string, to: string) { } describe.skipIf(!READY)("every proposed test, run by real dbt", () => { + const savedEnv = { + telemetry: process.env.ALTIMATE_TELEMETRY_DISABLED, + dbt: process.env.ALTIMATE_DBT_PATH, + } + let scratch: string | undefined + // Runs on failure too, so the copies of the database never outlive the test. + afterEach(() => { + if (scratch) fs.rmSync(scratch, { recursive: true, force: true }) + scratch = undefined + if (savedEnv.telemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = savedEnv.telemetry + if (savedEnv.dbt === undefined) delete process.env.ALTIMATE_DBT_PATH + else process.env.ALTIMATE_DBT_PATH = savedEnv.dbt + }) + test( "proposals are accepted by dbt, pass on clean data and fail on the corrupted copy", async () => { process.env.ALTIMATE_TELEMETRY_DISABLED = "true" process.env.ALTIMATE_DBT_PATH = DBT const root = fs.mkdtempSync(path.join(process.env.FI_VERIFY_WORK ?? os.tmpdir(), "fi-verify-")) + scratch = root const userProject = path.join(root, "user-project") const work = path.join(root, "work") const mutants = path.join(root, "mutants") @@ -345,7 +361,6 @@ describe.skipIf(!READY)("every proposed test, run by real dbt", () => { `[dbt-verify] ${report.project}: ${summary.proposals} proposals (${dedupe.size} distinct); dbt accepted ${summary.accepted_by_dbt}, ` + `passed on clean ${summary.passed_on_clean}, failed on corrupted ${summary.failed_on_corrupted}; ${withheld} slipped faults have no proposal (reason given)`, ) - fs.rmSync(root, { recursive: true, force: true }) // The point of the check: nothing the report presents may be unusable, and there is something to check. expect(summary.proposals).toBeGreaterThan(0) diff --git a/packages/opencode/test/altimate/fault-injection-driver.test.ts b/packages/opencode/test/altimate/fault-injection-driver.test.ts index c75b38d895..cfd7205d9d 100644 --- a/packages/opencode/test/altimate/fault-injection-driver.test.ts +++ b/packages/opencode/test/altimate/fault-injection-driver.test.ts @@ -656,6 +656,18 @@ describe("readDbtTarget", () => { } }) + test("an explicit profiles directory without profiles.yml is an error, not a fallback", async () => { + const { project, work, cleanup } = tempProject(twoTargets) + try { + fs.writeFileSync(path.join(project, "profiles.yml"), twoTargets) + const empty = path.join(work, "empty") + fs.mkdirSync(empty) + await expect(readDbtTarget(project, { profilesDir: empty, env: {} })).rejects.toThrow("No profiles.yml in the requested profiles directory") + } finally { + cleanup() + } + }) + test("a missing profile or target is named", async () => { const { project, cleanup } = tempProject("other:\n target: dev\n outputs: {}\n") try { @@ -901,12 +913,18 @@ describe.skipIf(process.platform === "win32")("createDbtRunner", () => { const controller = new AbortController() const pending = fake.runner.run("Baseline", ["build"], controller.signal) // Abort once the fake dbt is demonstrably running. + const started = Date.now() const poll = setInterval(() => { - if (!fs.existsSync(`${fake.argvFile}.cwd`)) return + // Bounded: if dbt never starts, abort anyway rather than keep the process alive. + if (!fs.existsSync(`${fake.argvFile}.cwd`) && Date.now() - started < 20_000) return clearInterval(poll) controller.abort() }, 10) - await expect(pending).rejects.toBeInstanceOf(FaultInjectionInterrupted) + try { + await expect(pending).rejects.toBeInstanceOf(FaultInjectionInterrupted) + } finally { + clearInterval(poll) + } // Already aborted: dbt is not started at all. fs.rmSync(fake.argvFile) await expect(fake.runner.run("Baseline", ["build"], controller.signal)).rejects.toBeInstanceOf(FaultInjectionInterrupted) @@ -996,6 +1014,12 @@ describe("fault-injection command", () => { expect(below.stderr).toContain("Catch rate 25.0% is below --fail-under 26.") }) + test("--fail-under compares exactly at the threshold despite floating-point rounding", async () => { + // 0.29 * 100 is 28.999999999999996 + expect((await run(okResult(0.29), { "fail-under": 29 })).exitCode).toBe(0) + expect((await run(okResult(0.57), { "fail-under": 57 })).exitCode).toBe(0) + }) + test("--fail-under cannot pass when nothing was measured", async () => { const out = await run(okResult(null), { "fail-under": 10 }) expect(out.exitCode).toBe(1) @@ -1199,6 +1223,25 @@ describe("report rendering", () => { expect(text).not.toContain("installed package") }) + test("a keyed comparison without key columns still renders", () => { + const keyless = { + ...slipped, + changed_relations: [ + { unique_id: "model.p.orders", baseline_rows: 99, rows: 99, comparison: { method: "keyed", rows_added: 0, rows_removed: 0, rows_changed: 5, columns: [] } }, + ], + } + const text = formatFaultInjection({ ...result, report: { ...result.report, slipped_through: [keyless] } } as any) + expect(text).toContain("5 rows changed of 99") + expect(text).not.toContain("matched on") + }) + + test("a node named like a YAML boolean or null is quoted", () => { + const proposal = { test: "unique", node_id: "model.p.true", resource_section: "models", column: "null", yaml: "- unique" } + const yaml = proposedTestYaml(proposal, { "model.p.true": { name: "true", resource_type: "model" } } as any, "1.10.0") + expect(yaml).toContain('- name: "true"') + expect(yaml).toContain('- name: "null"') + }) + test("row-level detail from a sampled comparison says what the per-column counts cover", () => { const sampled = { ...slipped, diff --git a/packages/opencode/test/altimate/fault-injection-e2e.test.ts b/packages/opencode/test/altimate/fault-injection-e2e.test.ts index 637f147865..8c39e34dab 100644 --- a/packages/opencode/test/altimate/fault-injection-e2e.test.ts +++ b/packages/opencode/test/altimate/fault-injection-e2e.test.ts @@ -72,6 +72,7 @@ describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () let database: string let hashBefore: string let savedDbtPath: string | undefined + const savedTelemetry = process.env.ALTIMATE_TELEMETRY_DISABLED beforeAll(() => { process.env.ALTIMATE_TELEMETRY_DISABLED = "true" @@ -94,7 +95,8 @@ describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () }, TIMEOUT_MS) afterAll(() => { - delete process.env.ALTIMATE_TELEMETRY_DISABLED + if (savedTelemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = savedTelemetry if (savedDbtPath === undefined) delete process.env.ALTIMATE_DBT_PATH else process.env.ALTIMATE_DBT_PATH = savedDbtPath if (root) fs.rmSync(root, { recursive: true, force: true }) @@ -217,7 +219,8 @@ describe.skipIf(!READY)("fault injection e2e (real engine, DuckDB and dbt)", () }) expect(asked).toHaveLength(1) expect(asked[0].permission).toBe("bash") - expect(asked[0].patterns).toEqual([`dbt build --project-dir ${fs.realpathSync(project)}`]) + expect(asked[0].patterns).toContain(`dbt build --project-dir ${fs.realpathSync(project)}`) + expect(asked[0].patterns).toHaveLength(5) expect(result.metadata.success).toBe(true) expect(result.metadata.executed).toBe(2) expect(result.metadata.killed).toBe(budgeted.report!.summary.killed) diff --git a/packages/opencode/test/altimate/fault-injection-registration.test.ts b/packages/opencode/test/altimate/fault-injection-registration.test.ts index caeccb596f..2861e523dc 100644 --- a/packages/opencode/test/altimate/fault-injection-registration.test.ts +++ b/packages/opencode/test/altimate/fault-injection-registration.test.ts @@ -54,7 +54,7 @@ function withoutEngine(body: Effect.Effect): Effect.Effect fs.rmSync(empty, { recursive: true, force: true }))) }) } @@ -78,7 +78,6 @@ describe("dbt_fault_injection registration", () => { const available = yield* Effect.promise(() => isFaultInjectionEngineAvailable()) const ids = yield* (yield* ToolRegistry.Service).ids() expect(ids.includes("dbt_fault_injection")).toBe(available) - if (process.env[CORE_DEV_PATH_ENV]) expect(available).toBe(true) }), ) }) diff --git a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts new file mode 100644 index 0000000000..bfea0cc9ff --- /dev/null +++ b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts @@ -0,0 +1,104 @@ +/** + * The `dbt_fault_injection` tool copies a project and runs its dbt code, so a project or profiles + * directory outside the session's workspace needs the `external_directory` permission the bash + * tool asks for, and relative paths resolve against the session directory. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import path from "path" +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import { runFaultInjection } from "../../src/altimate/native/connections/fault-injection" +import { DbtFaultInjectionTool } from "../../src/altimate/tools/dbt-fault-injection" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" +import { initTool } from "./tool-fixture" + +let received: Record | undefined + +beforeEach(async () => { + received = undefined + try { + await Dispatcher.call("__trigger_hook__" as any, {} as any) + } catch {} + Dispatcher.register("dbt.fault_injection", async (params: any) => { + received = params + return { success: true } + }) +}) + +afterEach(() => { + Dispatcher.register("dbt.fault_injection", (params: any) => runFaultInjection(params)) +}) + +async function runTool(directory: string, args: Record, deny = false) { + const asked: any[] = [] + const tool = await initTool(DbtFaultInjectionTool) + const outcome = await Instance.provide({ + directory, + fn: () => + tool + .execute(args, { + sessionID: "test", + messageID: "test", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async (request: any) => { + asked.push(request) + if (deny && request.permission === "external_directory") throw new Error("denied") + }, + }) + .then( + (result) => ({ result }), + (error) => ({ error }), + ), + }) + return { asked, error: (outcome as { error?: unknown }).error } +} + +describe("dbt_fault_injection permissions", () => { + test("a project inside the workspace needs only the bash permission, for the commands that really run", async () => { + await using tmp = await tmpdir() + const { asked } = await runTool(tmp.path, { project_dir: "dbt", budget: 1 }) + expect(asked.map((a) => a.permission)).toEqual(["bash"]) + const project = path.join(tmp.path, "dbt") + for (const command of ["parse", "compile", "build", "run", "test"]) { + expect(asked[0].patterns).toContain(`dbt ${command} --project-dir ${project}`) + } + expect(received?.project_dir).toBe(project) + }) + + test("a relative profiles_dir resolves against the session directory", async () => { + await using tmp = await tmpdir() + await runTool(tmp.path, { project_dir: ".", profiles_dir: "profiles", budget: 1 }) + expect(received?.profiles_dir).toBe(path.join(tmp.path, "profiles")) + }) + + test("a project outside the workspace asks for external_directory before anything runs", async () => { + await using tmp = await tmpdir() + await using outside = await tmpdir() + const { asked } = await runTool(tmp.path, { project_dir: outside.path, budget: 1 }) + expect(asked[0].permission).toBe("external_directory") + expect(asked[0].patterns).toEqual([path.join(outside.path, "*")]) + expect(received?.project_dir).toBe(outside.path) + }) + + test("a profiles directory outside the workspace asks too", async () => { + await using tmp = await tmpdir() + await using outside = await tmpdir() + const { asked } = await runTool(tmp.path, { project_dir: ".", profiles_dir: outside.path, budget: 1 }) + expect(asked.filter((a) => a.permission === "external_directory").map((a) => a.patterns[0])).toEqual([ + path.join(outside.path, "*"), + ]) + }) + + test("when external_directory is denied nothing is copied or run", async () => { + await using tmp = await tmpdir() + await using outside = await tmpdir() + const { asked, error } = await runTool(tmp.path, { project_dir: outside.path, budget: 1 }, true) + expect(String(error)).toContain("denied") + expect(received).toBeUndefined() + expect(asked.map((a) => a.permission)).toEqual(["external_directory"]) + }) +}) From 50b951e73d17db95a244a7acc140ad64d5c0209b Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 22:15:59 -0700 Subject: [PATCH 4/7] fix: [fault-injection] refuse project symlinks that leave the project and gate the default profile lookup - refuse a symlink whose target is outside the project, in the scratch tree, or a directory that contains the link; links inside the project are still copied as content - ask `external_directory` for the profiles directory dbt would pick by default (`DBT_PROFILES_DIR`, `~/.dbt`), not only for an explicit one - treat a dbt process killed by a signal as a failed run - tests for each, including the unit-test exclusion and relation-missing cases from the earlier review round Co-Authored-By: Claude Opus 5.5 (1M context) --- .../native/connections/fault-injection.ts | 78 ++++++++++++------- .../src/altimate/tools/dbt-fault-injection.ts | 14 +++- .../altimate/fault-injection-driver.test.ts | 54 +++++++++++++ 3 files changed, 115 insertions(+), 31 deletions(-) diff --git a/packages/opencode/src/altimate/native/connections/fault-injection.ts b/packages/opencode/src/altimate/native/connections/fault-injection.ts index 1dbf45b4e4..e57183eb3c 100644 --- a/packages/opencode/src/altimate/native/connections/fault-injection.ts +++ b/packages/opencode/src/altimate/native/connections/fault-injection.ts @@ -959,7 +959,8 @@ async function runDbtAction( } function dbtCrashed(outcome: DbtOutcome): boolean { - return outcome.exitCode !== null && outcome.exitCode > 1 + // A null code means dbt was killed by a signal. + return outcome.exitCode === null || outcome.exitCode > 1 } /** Perform one engine action and return the result to step the session with. */ @@ -1095,6 +1096,22 @@ export interface DbtTargetInfo { profilesFile: string } +/** + * The profiles.yml dbt would use: the explicit directory, else DBT_PROFILES_DIR, the project and ~/.dbt. + * An explicit directory is never replaced by a default, so a typo cannot select another profile. + */ +export function locateProfilesFile(projectDir: string, profilesDir?: string, env: NodeJS.ProcessEnv = process.env): string { + if (profilesDir) { + const explicit = path.join(path.resolve(profilesDir), "profiles.yml") + if (!fs.existsSync(explicit)) throw new Error(`No profiles.yml in the requested profiles directory (${explicit}).`) + return explicit + } + const candidates = [env.DBT_PROFILES_DIR, projectDir, path.join(os.homedir(), ".dbt")].filter((d): d is string => Boolean(d)) + const found = candidates.map((dir) => path.join(path.resolve(dir), "profiles.yml")).find((file) => fs.existsSync(file)) + if (!found) throw new Error(`No profiles.yml found (looked in ${candidates.join(", ")}).`) + return found +} + /** Read the profile and target a dbt project would use, in dbt's lookup order. */ export async function readDbtTarget( projectDir: string, @@ -1110,25 +1127,7 @@ export async function readDbtTarget( const profileName = String(env.DBT_PROFILE || resolveEnvVars(project?.profile ?? "", env)) if (!profileName) throw new Error(`${projectFile} does not name a profile.`) - const candidates = [ - options.profilesDir, - env.DBT_PROFILES_DIR, - projectDir, - path.join(os.homedir(), ".dbt"), - ].filter((d): d is string => Boolean(d)) - // An explicit directory is never replaced by a default: a typo must not select another profile. - const explicit = options.profilesDir ? path.join(path.resolve(options.profilesDir), "profiles.yml") : undefined - if (explicit && !fs.existsSync(explicit)) { - throw new Error(`No profiles.yml in the requested profiles directory (${explicit}).`) - } - const profilesFile = - explicit ?? - candidates - .map((dir) => path.join(path.resolve(dir), "profiles.yml")) - .find((file) => fs.existsSync(file)) - if (!profilesFile) { - throw new Error(`No profiles.yml found (looked in ${candidates.join(", ")}).`) - } + const profilesFile = locateProfilesFile(projectDir, options.profilesDir, env) const profiles = YAML.parse(await fsp.readFile(profilesFile, "utf-8")) as Record | null const profile = profiles?.[profileName] if (!profile || typeof profile !== "object") { @@ -1232,6 +1231,14 @@ export function supportsUnitTests(version: string): boolean { /** Top-level entries of a project that dbt does not need and that must not be copied. */ const PROJECT_COPY_SKIP = new Set([".git", "target", "logs", "node_modules", "profiles.yml", ".user.yml"]) +function isSymlink(file: string): boolean { + try { + return fs.lstatSync(file).isSymbolicLink() + } catch { + return false + } +} + /** * Copy the project into the work directory. dbt then runs in the copy, so * nothing it writes with a relative path (target/, logs/, a hook's export) @@ -1241,26 +1248,37 @@ export async function copyProject(projectDir: string, dest: string, workDir: str const [root, work] = await Promise.all([fsp.realpath(projectDir), fsp.realpath(workDir)]) await fsp.cp(root, dest, { recursive: true, - // Copy what a symlink points at: a link kept as a link would lead dbt back to the user's files. + // Copy what a link inside the project points at: a link kept as a link would lead dbt back to the + // user's files. Links that leave the project, or lead back up into it, are refused below. dereference: true, mode: fs.constants.COPYFILE_FICLONE, filter: (source) => { if (source === work || source.startsWith(work + path.sep)) return false - // A link is followed by the copy; refuse one that leads into, or around, the scratch tree. - try { - if (fs.lstatSync(source).isSymbolicLink()) { - const target = fs.realpathSync(source) - if (target === work || target.startsWith(work + path.sep) || work.startsWith(target + path.sep)) return false - } - } catch { - return false // dangling link - } const relative = path.relative(root, source) if (relative === "") return true if (!relative.includes(path.sep) && PROJECT_COPY_SKIP.has(relative)) return false if (/\.(duckdb|wal)$/i.test(source)) return false // A Python virtualenv, whatever it is called. if (fs.existsSync(path.join(source, "pyvenv.cfg"))) return false + if (isSymlink(source)) { + let target: string + try { + target = fs.realpathSync(source) + } catch { + return false // dangling link + } + const inside = target === root || target.startsWith(root + path.sep) + if (!inside || target === work || target.startsWith(work + path.sep)) { + throw new Error( + `Refusing to run: ${source} is a symbolic link to ${target}, outside the project, and copying it would read files the project does not contain. Replace the link with the files it points at, or run from a project without it.`, + ) + } + // A link to a directory that holds the link itself would be copied again inside its own copy. + const parent = fs.realpathSync(path.dirname(source)) + if (fs.statSync(target).isDirectory() && (parent === target || parent.startsWith(target + path.sep))) { + throw new Error(`Refusing to run: ${source} is a symbolic link to ${target}, a directory that contains it.`) + } + } return true }, }) diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts index c5c95fd7b0..3fc7ca9667 100644 --- a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -3,6 +3,7 @@ import path from "path" import { Tool } from "../../tool/tool" import { Instance } from "../../project/instance" import { assertExternalDirectoryLegacy } from "../../tool/external-directory" +import { locateProfilesFile } from "../native/connections/fault-injection" import { Dispatcher } from "../native" import { formatFaultInjection, summarizeFaultInjection } from "../native/connections/fault-injection-report" @@ -59,7 +60,18 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { // The project is copied and its dbt code is run, so a path outside the workspace needs the same // external_directory permission the bash tool asks for. await assertExternalDirectoryLegacy(ctx, projectDir, { kind: "directory" }) - if (profilesDir) await assertExternalDirectoryLegacy(ctx, profilesDir, { kind: "directory" }) + // The profile dbt will use: the explicit directory, else its default lookup (DBT_PROFILES_DIR, ~/.dbt). + if (profilesDir) { + await assertExternalDirectoryLegacy(ctx, profilesDir, { kind: "directory" }) + } else { + let located: string | undefined + try { + located = locateProfilesFile(projectDir) + } catch { + // No profile found anywhere: the run reports that itself. + } + if (located) await assertExternalDirectoryLegacy(ctx, path.dirname(located), { kind: "directory" }) + } // The run executes these dbt commands on the copies; ask for each rather than for a proxy. const commands = ["parse", "compile", "build", "run", "test"].map((c) => `dbt ${c} --project-dir ${projectDir}`) await ctx.ask({ diff --git a/packages/opencode/test/altimate/fault-injection-driver.test.ts b/packages/opencode/test/altimate/fault-injection-driver.test.ts index cfd7205d9d..21a0c9f674 100644 --- a/packages/opencode/test/altimate/fault-injection-driver.test.ts +++ b/packages/opencode/test/altimate/fault-injection-driver.test.ts @@ -769,6 +769,60 @@ describe("DuckDbSandbox.assertManifestIsolated", () => { expect(sandbox.isRelationMissing(new Error("Binder Error: column not found in Catalog Error: text"))).toBe(false) }) + test("only a missing table, view or schema counts as a missing relation", () => { + expect(sandbox.isRelationMissing(new Error("Catalog Error: Scalar Function with name nofunc does not exist!"))).toBe(false) + expect(sandbox.isRelationMissing(new Error("Catalog Error: Type with name NOTYPE does not exist!"))).toBe(false) + expect(sandbox.isRelationMissing(new Error("Catalog Error: View with name v does not exist!"))).toBe(true) + expect(sandbox.isRelationMissing(new Error("Catalog Error: Schema with name s does not exist!"))).toBe(true) + }) + + test("unit tests are excluded from the baseline build only on dbt 1.8 and newer", () => { + expect(supportsUnitTests("1.7.14")).toBe(false) + expect(supportsUnitTests("1.8.0")).toBe(true) + expect(supportsUnitTests("1.11.7")).toBe(true) + expect(supportsUnitTests("2.0.0")).toBe(true) + expect(supportsUnitTests("unknown")).toBe(true) + }) + + test("symlinks inside the project are copied as content; links that leave it or loop are refused", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-copy-")) + try { + const outside = path.join(root, "outside") + const project = path.join(root, "project") + const work = path.join(root, "work") + for (const d of [outside, project, work, path.join(project, "models")]) fs.mkdirSync(d) + fs.writeFileSync(path.join(outside, "seed.csv"), "id\n1\n") + fs.writeFileSync(path.join(project, "real.csv"), "id\n2\n") + fs.symlinkSync("real.csv", path.join(project, "alias.csv")) + fs.symlinkSync(path.join(project, "real.csv"), path.join(project, "models", "abs.csv")) + fs.symlinkSync(path.join(project, "nowhere.csv"), path.join(project, "dangling.csv")) + // Inside node_modules a link is skipped with its directory, not inspected. + fs.mkdirSync(path.join(project, "node_modules")) + fs.symlinkSync(outside, path.join(project, "node_modules", "dep")) + const copy = path.join(work, "copy") + await copyProject(project, copy, work) + for (const name of ["alias.csv", path.join("models", "abs.csv")]) { + expect(fs.lstatSync(path.join(copy, name)).isSymbolicLink()).toBe(false) + expect(fs.readFileSync(path.join(copy, name), "utf-8")).toBe("id\n2\n") + } + expect(fs.existsSync(path.join(copy, "dangling.csv"))).toBe(false) + + // A link outside the project (absolute or relative), or to a parent of the project or the scratch tree, is refused. + for (const target of [path.join(outside, "seed.csv"), "../../outside/seed.csv", root, work]) { + const link = path.join(project, "models", "bad") + fs.symlinkSync(target, link) + await expect(copyProject(project, path.join(work, "copy2"), work)).rejects.toThrow("Refusing to run") + fs.unlinkSync(link) + fs.rmSync(path.join(work, "copy2"), { recursive: true, force: true }) + } + // A link back up to a directory that contains it would copy the project into itself. + fs.symlinkSync("..", path.join(project, "models", "back")) + await expect(copyProject(project, path.join(work, "copy3"), work)).rejects.toThrow("a directory that contains it") + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } + }) + test("a database named like a reserved DuckDB catalog is refused", () => { expect( () => From 41ee5ff44a2db74ee705ed3ac5ef7a4a8655748b Mon Sep 17 00:00:00 2001 From: Anand Gupta Date: Sun, 4 Oct 2026 22:34:07 -0700 Subject: [PATCH 5/7] fix: [fault-injection] refuse directory symlinks, gate a linked default profile, tighten relation-missing match - refuse every symlink to a directory when copying the project, which also rules out cycles between two directories - authorize the real location of the default `profiles.yml`, so a link to a file outside the workspace is gated by where it points - match only `Table|View|Schema with name ... does not exist` as a missing relation, not `Table Function` - test: give the fake dbt runner a timeout that survives a loaded machine Co-Authored-By: Claude Opus 5.5 (1M context) --- .../native/connections/fault-injection.ts | 11 ++++++----- .../src/altimate/tools/dbt-fault-injection.ts | 3 ++- .../altimate/fault-injection-driver.test.ts | 18 +++++++++++------- .../fault-injection-tool-permission.test.ts | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/altimate/native/connections/fault-injection.ts b/packages/opencode/src/altimate/native/connections/fault-injection.ts index e57183eb3c..d41d34a1de 100644 --- a/packages/opencode/src/altimate/native/connections/fault-injection.ts +++ b/packages/opencode/src/altimate/native/connections/fault-injection.ts @@ -605,7 +605,7 @@ export class DuckDbSandbox implements SandboxStrategy { isRelationMissing(error: unknown): boolean { // Anywhere at a line start: the driver may prefix DuckDB's text with its own explanation. // Only a missing relation: a missing function or type is also a Catalog Error but is a broken query. - return /(^|\n)\s*(Error: )?Catalog Error:\s*(Table|View|Schema)\b[^\n]*does not exist/i.test(errorText(error)) + return /(^|\n)\s*(Error: )?Catalog Error:\s*(Table|View|Schema) with name[^\n]*does not exist/i.test(errorText(error)) } async listRelations(): Promise { @@ -1273,10 +1273,11 @@ export async function copyProject(projectDir: string, dest: string, workDir: str `Refusing to run: ${source} is a symbolic link to ${target}, outside the project, and copying it would read files the project does not contain. Replace the link with the files it points at, or run from a project without it.`, ) } - // A link to a directory that holds the link itself would be copied again inside its own copy. - const parent = fs.realpathSync(path.dirname(source)) - if (fs.statSync(target).isDirectory() && (parent === target || parent.startsWith(target + path.sep))) { - throw new Error(`Refusing to run: ${source} is a symbolic link to ${target}, a directory that contains it.`) + // Directory links can form cycles (a/x -> ../b, b/y -> ../a) that a dereferencing copy would follow forever. + if (fs.statSync(target).isDirectory()) { + throw new Error( + `Refusing to run: ${source} is a symbolic link to the directory ${target}. Replace it with the directory itself; links to files are copied as content.`, + ) } } return true diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts index 3fc7ca9667..f80432fc0c 100644 --- a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -1,4 +1,5 @@ import z from "zod" +import fs from "fs" import path from "path" import { Tool } from "../../tool/tool" import { Instance } from "../../project/instance" @@ -70,7 +71,7 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { } catch { // No profile found anywhere: the run reports that itself. } - if (located) await assertExternalDirectoryLegacy(ctx, path.dirname(located), { kind: "directory" }) + if (located) await assertExternalDirectoryLegacy(ctx, path.dirname(fs.realpathSync(located)), { kind: "directory" }) } // The run executes these dbt commands on the copies; ask for each rather than for a proxy. const commands = ["parse", "compile", "build", "run", "test"].map((c) => `dbt ${c} --project-dir ${projectDir}`) diff --git a/packages/opencode/test/altimate/fault-injection-driver.test.ts b/packages/opencode/test/altimate/fault-injection-driver.test.ts index 21a0c9f674..49753c7919 100644 --- a/packages/opencode/test/altimate/fault-injection-driver.test.ts +++ b/packages/opencode/test/altimate/fault-injection-driver.test.ts @@ -772,6 +772,7 @@ describe("DuckDbSandbox.assertManifestIsolated", () => { test("only a missing table, view or schema counts as a missing relation", () => { expect(sandbox.isRelationMissing(new Error("Catalog Error: Scalar Function with name nofunc does not exist!"))).toBe(false) expect(sandbox.isRelationMissing(new Error("Catalog Error: Type with name NOTYPE does not exist!"))).toBe(false) + expect(sandbox.isRelationMissing(new Error("Catalog Error: Table Function with name f does not exist!"))).toBe(false) expect(sandbox.isRelationMissing(new Error("Catalog Error: View with name v does not exist!"))).toBe(true) expect(sandbox.isRelationMissing(new Error("Catalog Error: Schema with name s does not exist!"))).toBe(true) }) @@ -784,7 +785,7 @@ describe("DuckDbSandbox.assertManifestIsolated", () => { expect(supportsUnitTests("unknown")).toBe(true) }) - test("symlinks inside the project are copied as content; links that leave it or loop are refused", async () => { + test("symlinks inside the project are copied as content; links that leave it or point at directories are refused", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-copy-")) try { const outside = path.join(root, "outside") @@ -815,9 +816,12 @@ describe("DuckDbSandbox.assertManifestIsolated", () => { fs.unlinkSync(link) fs.rmSync(path.join(work, "copy2"), { recursive: true, force: true }) } - // A link back up to a directory that contains it would copy the project into itself. - fs.symlinkSync("..", path.join(project, "models", "back")) - await expect(copyProject(project, path.join(work, "copy3"), work)).rejects.toThrow("a directory that contains it") + // Directory links are refused, including cycles between two directories. + fs.mkdirSync(path.join(project, "a")) + fs.mkdirSync(path.join(project, "b")) + fs.symlinkSync("../b", path.join(project, "a", "to-b")) + fs.symlinkSync("../a", path.join(project, "b", "to-a")) + await expect(copyProject(project, path.join(work, "copy3"), work)).rejects.toThrow("symbolic link to the directory") } finally { fs.rmSync(root, { recursive: true, force: true }) } @@ -844,7 +848,7 @@ describe("DuckDbSandbox.assertManifestIsolated", () => { describe.skipIf(process.platform === "win32")("createDbtRunner", () => { /** A stand-in for dbt: a shell script whose behaviour is chosen by FAKE_DBT_MODE. */ - function fakeDbt(): { runner: ReturnType; root: string; argvFile: string; cleanup: () => void; mode: (m: string) => void } { + function fakeDbt(timeoutMs = 10_000): { runner: ReturnType; root: string; argvFile: string; cleanup: () => void; mode: (m: string) => void } { const root = fs.mkdtempSync(path.join(os.tmpdir(), "fi-dbt-runner-")) const script = path.join(root, "dbt") const argvFile = path.join(root, "argv.txt") @@ -874,7 +878,7 @@ describe.skipIf(process.platform === "win32")("createDbtRunner", () => { workDir: root, dbtProfile: "p", dbtTarget: "dev", - timeoutMs: 400, + timeoutMs, profilesDir: (target) => path.join(root, `profiles-${target.toLowerCase()}`), }) return { @@ -946,7 +950,7 @@ describe.skipIf(process.platform === "win32")("createDbtRunner", () => { }) test("a run that exceeds the timeout is stopped and reported as timed out", async () => { - const fake = fakeDbt() + const fake = fakeDbt(400) try { fake.mode("hang") const started = Date.now() diff --git a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts index bfea0cc9ff..4eeeadbd17 100644 --- a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts +++ b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts @@ -5,6 +5,7 @@ */ import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import fs from "fs" import path from "path" import * as Dispatcher from "../../src/altimate/native/dispatcher" import { runFaultInjection } from "../../src/altimate/native/connections/fault-injection" @@ -93,6 +94,24 @@ describe("dbt_fault_injection permissions", () => { ]) }) + test("a profiles.yml that is a link to a file outside the workspace is gated by its real location", async () => { + await using tmp = await tmpdir() + await using outside = await tmpdir() + await Bun.write(path.join(outside.path, "profiles.yml"), "p: {}\n") + await Bun.write(path.join(tmp.path, "dbt_project.yml"), "name: p\nprofile: p\n") + fs.symlinkSync(path.join(outside.path, "profiles.yml"), path.join(tmp.path, "profiles.yml")) + const saved = process.env.DBT_PROFILES_DIR + delete process.env.DBT_PROFILES_DIR + try { + const { asked } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) + expect(asked.filter((a) => a.permission === "external_directory").map((a) => a.patterns[0])).toEqual([ + path.join(fs.realpathSync(outside.path), "*"), + ]) + } finally { + if (saved !== undefined) process.env.DBT_PROFILES_DIR = saved + } + }) + test("when external_directory is denied nothing is copied or run", async () => { await using tmp = await tmpdir() await using outside = await tmpdir() From a44605a0459b1862f8d9d76b41669eed80ce072c Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 22:46:31 -0700 Subject: [PATCH 6/7] fix: [fault-injection] report an unreadable default profile as an error result A `profiles.yml` that disappears or is a dangling link between lookup and `realpath` made the tool reject instead of returning its error result. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/altimate/tools/dbt-fault-injection.ts | 17 ++++++++++++----- .../fault-injection-tool-permission.test.ts | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts index f80432fc0c..eb683ce326 100644 --- a/packages/opencode/src/altimate/tools/dbt-fault-injection.ts +++ b/packages/opencode/src/altimate/tools/dbt-fault-injection.ts @@ -65,13 +65,20 @@ export const DbtFaultInjectionTool = Tool.define("dbt_fault_injection", { if (profilesDir) { await assertExternalDirectoryLegacy(ctx, profilesDir, { kind: "directory" }) } else { - let located: string | undefined + let profileDir: string | undefined try { - located = locateProfilesFile(projectDir) - } catch { - // No profile found anywhere: the run reports that itself. + profileDir = path.dirname(fs.realpathSync(locateProfilesFile(projectDir))) + } catch (e) { + // A profile that is missing or an unreadable link: report it as the run would, without + // dispatching and without skipping the authorization above. + const msg = e instanceof Error ? e.message : String(e) + return { + title: "Fault injection: ERROR", + metadata: { success: false, error: msg }, + output: `Fault injection failed: ${msg}`, + } } - if (located) await assertExternalDirectoryLegacy(ctx, path.dirname(fs.realpathSync(located)), { kind: "directory" }) + await assertExternalDirectoryLegacy(ctx, profileDir, { kind: "directory" }) } // The run executes these dbt commands on the copies; ask for each rather than for a proxy. const commands = ["parse", "compile", "build", "run", "test"].map((c) => `dbt ${c} --project-dir ${projectDir}`) diff --git a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts index 4eeeadbd17..e6c3bd057f 100644 --- a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts +++ b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts @@ -61,6 +61,7 @@ async function runTool(directory: string, args: Record, deny = describe("dbt_fault_injection permissions", () => { test("a project inside the workspace needs only the bash permission, for the commands that really run", async () => { await using tmp = await tmpdir() + await Bun.write(path.join(tmp.path, "dbt", "profiles.yml"), "p: {}\n") const { asked } = await runTool(tmp.path, { project_dir: "dbt", budget: 1 }) expect(asked.map((a) => a.permission)).toEqual(["bash"]) const project = path.join(tmp.path, "dbt") @@ -79,6 +80,7 @@ describe("dbt_fault_injection permissions", () => { test("a project outside the workspace asks for external_directory before anything runs", async () => { await using tmp = await tmpdir() await using outside = await tmpdir() + await Bun.write(path.join(outside.path, "profiles.yml"), "p: {}\n") const { asked } = await runTool(tmp.path, { project_dir: outside.path, budget: 1 }) expect(asked[0].permission).toBe("external_directory") expect(asked[0].patterns).toEqual([path.join(outside.path, "*")]) @@ -112,6 +114,22 @@ describe("dbt_fault_injection permissions", () => { } }) + test("a dangling profiles.yml link is reported as an error result, not thrown", async () => { + await using tmp = await tmpdir() + await Bun.write(path.join(tmp.path, "dbt_project.yml"), "name: p\nprofile: p\n") + fs.symlinkSync(path.join(tmp.path, "missing.yml"), path.join(tmp.path, "profiles.yml")) + const saved = process.env.DBT_PROFILES_DIR + delete process.env.DBT_PROFILES_DIR + try { + const { asked, error } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) + expect(error).toBeUndefined() + expect(asked).toEqual([]) + expect(received).toBeUndefined() + } finally { + if (saved !== undefined) process.env.DBT_PROFILES_DIR = saved + } + }) + test("when external_directory is denied nothing is copied or run", async () => { await using tmp = await tmpdir() await using outside = await tmpdir() From 50f9a2aec842499aa3620772def3b19d5efc54fc Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 4 Oct 2026 23:02:04 -0700 Subject: [PATCH 7/7] test: [fault-injection] isolate the tool permission tests from DBT_PROFILES_DIR and assert the error result Co-Authored-By: Claude Opus 5.5 (1M context) --- .../fault-injection-tool-permission.test.ts | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts index e6c3bd057f..3288eae04a 100644 --- a/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts +++ b/packages/opencode/test/altimate/fault-injection-tool-permission.test.ts @@ -31,7 +31,20 @@ afterEach(() => { Dispatcher.register("dbt.fault_injection", (params: any) => runFaultInjection(params)) }) -async function runTool(directory: string, args: Record, deny = false) { +/** Runs the tool with DBT_PROFILES_DIR set to `profilesEnv` (unset by default) so the host's value never leaks in. */ +async function runTool(directory: string, args: Record, deny = false, profilesEnv?: string) { + const saved = process.env.DBT_PROFILES_DIR + if (profilesEnv === undefined) delete process.env.DBT_PROFILES_DIR + else process.env.DBT_PROFILES_DIR = profilesEnv + try { + return await runToolInner(directory, args, deny) + } finally { + if (saved === undefined) delete process.env.DBT_PROFILES_DIR + else process.env.DBT_PROFILES_DIR = saved + } +} + +async function runToolInner(directory: string, args: Record, deny: boolean) { const asked: any[] = [] const tool = await initTool(DbtFaultInjectionTool) const outcome = await Instance.provide({ @@ -55,7 +68,7 @@ async function runTool(directory: string, args: Record, deny = (error) => ({ error }), ), }) - return { asked, error: (outcome as { error?: unknown }).error } + return { asked, error: (outcome as { error?: unknown }).error, result: (outcome as { result?: any }).result } } describe("dbt_fault_injection permissions", () => { @@ -96,38 +109,38 @@ describe("dbt_fault_injection permissions", () => { ]) }) + test("the default profile lookup is gated too when it leads outside the workspace", async () => { + await using tmp = await tmpdir() + await using outside = await tmpdir() + await Bun.write(path.join(outside.path, "profiles.yml"), "p: {}\n") + const { asked } = await runTool(tmp.path, { project_dir: ".", budget: 1 }, false, outside.path) + expect(asked.filter((a) => a.permission === "external_directory").map((a) => a.patterns[0])).toEqual([ + path.join(outside.path, "*"), + ]) + }) + test("a profiles.yml that is a link to a file outside the workspace is gated by its real location", async () => { await using tmp = await tmpdir() await using outside = await tmpdir() await Bun.write(path.join(outside.path, "profiles.yml"), "p: {}\n") await Bun.write(path.join(tmp.path, "dbt_project.yml"), "name: p\nprofile: p\n") fs.symlinkSync(path.join(outside.path, "profiles.yml"), path.join(tmp.path, "profiles.yml")) - const saved = process.env.DBT_PROFILES_DIR - delete process.env.DBT_PROFILES_DIR - try { - const { asked } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) - expect(asked.filter((a) => a.permission === "external_directory").map((a) => a.patterns[0])).toEqual([ - path.join(fs.realpathSync(outside.path), "*"), - ]) - } finally { - if (saved !== undefined) process.env.DBT_PROFILES_DIR = saved - } + const { asked } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) + expect(asked.filter((a) => a.permission === "external_directory").map((a) => a.patterns[0])).toEqual([ + path.join(fs.realpathSync(outside.path), "*"), + ]) }) test("a dangling profiles.yml link is reported as an error result, not thrown", async () => { await using tmp = await tmpdir() await Bun.write(path.join(tmp.path, "dbt_project.yml"), "name: p\nprofile: p\n") fs.symlinkSync(path.join(tmp.path, "missing.yml"), path.join(tmp.path, "profiles.yml")) - const saved = process.env.DBT_PROFILES_DIR - delete process.env.DBT_PROFILES_DIR - try { - const { asked, error } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) - expect(error).toBeUndefined() - expect(asked).toEqual([]) - expect(received).toBeUndefined() - } finally { - if (saved !== undefined) process.env.DBT_PROFILES_DIR = saved - } + const { asked, error, result } = await runTool(tmp.path, { project_dir: ".", budget: 1 }) + expect(error).toBeUndefined() + expect(result.title).toBe("Fault injection: ERROR") + expect(result.metadata.success).toBe(false) + expect(asked).toEqual([]) + expect(received).toBeUndefined() }) test("when external_directory is denied nothing is copied or run", async () => {